Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_output>
%matplotlib inline
class TweetDataset:

 def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<load_pretrained>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 tr = pd.concat([df_train.drop(['Survived'],axis=1), df_test])
 tr.index = tr['PassengerId']
tr.head()
class TweetModel(transformers.BertPreTrainedModel):

 def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
tr[tr.columns[tr.isnull().any() ]].isnull().sum()
def loss_fn(start_logits, end_logits, start_positions, end_positions):

 loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss<choose_model_class>
tr_age_mean = tr.groupby('NameTitle')['Age'].mean()
tr_age_null = tr['Age'].isnull()
tr_age_mean_1 = tr_age_mean[tr.loc[tr_age_null, 'NameTitle']]
tr_age_mean_1.index = tr[tr_age_null].index
tr_age_mean_1
tr.loc[tr_age_null, 'Age'] = tr_age_mean_1
tr[tr['Age'].isnull() ]
def train_fn(data_loader, model, optimizer, device, scheduler=None):

 model.train()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg )<compute_test_metric>
tr.loc[tr['Fare'].isnull() , 'Fare'] = tr.query('Age>60 & Pclass==3')['Fare'].mean()
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):

 if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):

 model.eval()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg<load_from_csv>
def dummy_to_pca(tr, column_name:str, features):
max_seq = 300
max_d = 15
col_count = tr.groupby(column_name)[column_name].count()
if len(col_count)> max_seq:
tops = col_count.sort_values(ascending=False)[0:max_seq].index
f =tr.loc[tr[column_name].isin(tops)][['PassengerId', column_name]]
else:
tops = col_count.index
f =tr[['PassengerId', column_name]]
f = pd.get_dummies(f, columns=[column_name])
 f = f.groupby('PassengerId' ).mean()
if len(tops)< max_d:
max_d = len(tops)
 pca = PCA(n_components=max_d)
 pca.fit(f)
 cumsum = np.cumsum(pca.explained_variance_ratio_)
 num_d = np.argmax(cumsum >= 0.99)+ 1
if num_d == 1:
num_d = max_d
pca = PCA(n_components=num_d)
 result = pca.fit_transform(f)
 result = pd.DataFrame(result)
 result.columns = [column_name + '_' + str(column)for column in result.columns]
result.index = f.index
return pd.concat([features, result], axis=1, join_axes=[features.index] )
def run(fold):

 dfx = pd.read_csv(TRAINING_FILE)
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
num_workers=2 
)
 device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
model = TweetModel(conf=model_config)
 model.to(device)
 num_train_steps = int(len(df_train)/ TRAIN_BATCH_SIZE * EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 es = utils.EarlyStopping(patience=2, mode="max")
 print(f"Training is Starting for fold={fold}")
 for epoch in range(3):
train_fn(train_data_loader, model, optimizer, device, scheduler=scheduler)
 jaccard = eval_fn(valid_data_loader, model, device)
 print(f"Jaccard Score = {jaccard}")
 es(jaccard, model, model_path=f"model_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break<load_from_csv>
def mean_encoding(tr, feature_name):
mean = df_train.groupby(feature_name)['Survived'].mean()
tr.loc[:,feature_name] = tr[feature_name].map(mean)
 tr.loc[tr[feature_name].isnull() , feature_name] = df_train['Survived'].mean()
fold=0
dfx = pd.read_csv(TRAINING_FILE)
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
num_workers=2
)<load_from_csv>
def create_ticket_label(tr):
tr['TicketLabel'] = tr['Ticket'].str.extract(r'([A-Za-z0-9/.]+)')
 tr['TicketLabel'] = tr['TicketLabel'].str.replace("\.", "")
 tr['TicketLabel'] = tr['TicketLabel'].str.replace("/", "")
 tr['TicketLabel'] = tr['TicketLabel'].str.upper()
tr['TicketLabel'].replace(['CASOTON','SCOW', 'AQ3', 'AQ4', 'SOP', 'STONOQ', 'STONO2', 'SCA3', 'A'],
['CA', 'SC', 'AQ', 'AQ', 'SOPP', 'SOTONOQ', 'SOTONO2', 'SC', 'A4'],inplace=True)
 tr['TicketLabel'].fillna('NaN', inplace=True)
 create_ticket_label(df_train)
 create_ticket_label(tr)
 df_train.groupby('TicketLabel')['Survived'].agg(["mean", "size"] )
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
def create_ticket_a(tr):
tr['TicketA'] = tr['TicketLabel'].str[:2]
tr['TicketA'].fillna('NaN', inplace=True)
 create_ticket_a(tr)
 create_ticket_a(df_train)
 df_train.groupby('TicketA')['Survived'].agg(["mean", "size"] )
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True<load_pretrained>
def create_cabin_a(tr):
tr['CabinA'] = tr['Cabin'].str[:1]
tr['CabinB'] = tr['Cabin'].str[:2]
tr['CabinA'].fillna('NaN', inplace=True)
 tr['CabinB'].fillna('NaN', inplace=True)
 create_cabin_a(tr)
 create_cabin_a(df_train)
 df_train.groupby(['CabinA','CabinB'])['Survived'].agg(["mean", "size"] )
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load("model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load("model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load("model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load("model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load("model_4.bin"))
model5.eval()<create_dataframe>
def create_family_size(tr):
tr['FamilySize'] = tr['SibSp'] + tr['Parch'] + 1
tr['IsAlone'] = tr['FamilySize'] == 1
create_family_size(tr)
 create_family_size(df_train)
 df_train.groupby('FamilySize')['Survived'].agg(["mean", "size"])
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(
 outputs_start1
+ outputs_start2
+ outputs_start3
+ outputs_start4
+ outputs_start5 
)/ 5
outputs_end =(
 outputs_end1
+ outputs_end2
+ outputs_end3
+ outputs_end4
+ outputs_end5 
)/ 5
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<string_transform>
def get_features_mean() :
f = tr[['PassengerId','Pclass','Age', 'SibSp', 'Parch', 'Fare', 'Embarked',
'FamilySize', 'IsAlone', 'NameTitle','CabinA', 'CabinB', 'AgeBand',
'Sex', 'TicketA', 'TicketLabel']]
f.index = f['PassengerId']
mean_encoding(f, 'Sex')
 mean_encoding(f, 'IsAlone')
 mean_encoding(f, 'AgeBand')
 mean_encoding(f, 'Embarked')
 mean_encoding(f, 'NameTitle')
 mean_encoding(f, 'CabinA')
 mean_encoding(f, 'CabinB')
 mean_encoding(f, 'TicketA')
 mean_encoding(f, 'TicketLabel')
 scaler = pp.StandardScaler()
f = pd.DataFrame(scaler.fit_transform(f), columns=f.columns)
 f['PassengerId'] = tr.index
return f
def post_process(selected):
return " ".join(set(selected.lower().split()))<save_to_csv>
def get_features_pca() :
f = tr[['PassengerId','Pclass','Age', 'SibSp', 'Parch', 'Fare', 'Embarked',
'FamilySize', 'IsAlone', 'NameTitle','CabinA', 'Sex', 'TicketA']]
f.index = f['PassengerId']
f = pd.get_dummies(f, columns=['Sex', 'IsAlone','Embarked', 'NameTitle', 'CabinA', 'TicketA'])
 f = dummy_to_pca(tr, 'TicketLabel', f)
 f = dummy_to_pca(tr, 'AgeBand', f)
 f = dummy_to_pca(tr, 'CabinB', f)
 scaler = pp.StandardScaler()
f = pd.DataFrame(scaler.fit_transform(f), columns=f.columns)
 f['PassengerId'] = tr.index
return f
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.selected_text = sample.selected_text.map(post_process)
 sample.to_csv("submission.csv", index=False )<set_options>
def split_train_data(f):
X_train = df_train[['PassengerId']]
X_train = pd.merge(X_train, f, how='left')
 y_train = df_train.Survived
X_test = df_test[['PassengerId']]
X_test = pd.merge(X_test, f, how='left')
 X_train.drop(['PassengerId'], axis=1, inplace=True)
 X_test.drop(['PassengerId'], axis=1, inplace=True)
 return X_train, y_train, X_test
batch_size = 8
n_epochs = 2
max_context_len = 150
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print("device: ", device )<create_dataframe>
if not sys.warnoptions:
warnings.simplefilter("ignore")
 def train(X_train, y_train):
clf_mlp = MLPClassifier()
clf_xb = XGBClassifier()
clf_rf = RandomForestClassifier()
clfs = [ 
('xgb', clf_xb), 
('rf', clf_rf), 
('mlp', clf_mlp),
]
clf_eb = VotingClassifier(estimators=clfs, voting='soft')
 parameters = {
'xgb__max_depth':[4], 'xgb__min_child_weight':[4], 'xgb__gamma':[0.2],
'xgb__subsample':[0.9], 'xgb__colsample_bytree':[0.84],
'xgb__reg_alpha':[0.01], 'xgb__learning_rate':[0.2],
"rf__n_estimators":[45], "rf__max_depth":[20], "rf__min_samples_leaf":[3],
'mlp__solver':['adam'], 'mlp__max_iter':[1000], 'mlp__early_stopping':[True],
'mlp__hidden_layer_sizes':[(128,64,32)],'mlp__activation':['logistic'],
}
clf = GridSearchCV(clf_eb, parameters, n_jobs=-1, cv=5)
 clf.fit(X_train, y_train)
 score = cross_val_score(clf, X_train, y_train, cv=5, scoring='accuracy')
 print("Accuracy: %0.4f(+/- %0.4f)[%s]" %(score.mean() , score.std() , "eb"))
return score.mean() , clf
def divide_dataset(__train, factor, fields):
train_size = int(len(__train)* factor)
 valid_size = len(__train)- train_size
valid_indices = list(range(train_size+valid_size))
random.shuffle(valid_indices)
 train_indices = valid_indices[:train_size]
valid_indices = valid_indices[train_size:]
train = [__train[idx] for idx in train_indices]
valid = [__train[idx] for idx in valid_indices]
train = data.Dataset(train, fields)
 valid = data.Dataset(valid, fields)
 return train, valid
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" \
or ord(c)== 0x202F:
return True
return False
def character_filter(c):
if c == "\t": return ""
if ord(c)<128: return c
if c in "≠•∞™ˈʃʊʁʁiʁɑ̃ʃɔ.̃ºª¶§¡£¢ç": return "z"
if c in "àáâãäåæ": return "a"
if c in "èéêë": return "e"
if c in "ìíîï": return "i"
if c in "òóôõöōŏő": return "o"
if c in "ùúûü": return "u"
if c in "ÀÁÂÃÄÅ": return "A"
if c in "ÈÉÊË": return "E"
if c in "ÌÍÎÏ": return "I"
if c in "ÒÓÔÕÖŌŎŐ": return "O"
if c in "ÙÚÛÜ": return "U"
return "z"
def postproc(text, x):
ret = []
for sent in text:
ret.append(tokenizer.convert_tokens_to_ids(sent))
return ret<load_from_csv>
def create_data_n_train(data_type):
if(data_type == 'PCA'):
features = get_features_pca()
else:
features = get_features_mean()
X_train, y_train, X_test = split_train_data(features)
 score, clf = train(X_train, y_train)
 pred = clf.fit(X_train, y_train ).predict(X_test)
 return score, pred
def read_SQuAD(file, fields, train):
input_data = pd.read_csv(file)
 examples = []
word_to_char_list = []
context_raw_test = []
for idx, row in input_data.iterrows() :
context_raw = row['text']
question_raw = row['sentiment']
if type(context_raw)!= str:
print("data", idx, "is illegal")
 continue
context_raw = context_raw.strip()
if not train:
context_raw_test.append(context_raw)
 context_raw = "".join(map(character_filter, context_raw))
pt = 0
char_to_word = [-1]*len(context_raw)
 word_to_char = []
context = tokenizer.tokenize(context_raw)
 question = tokenizer.tokenize(question_raw)
 word_to_char = [-1]*(len(question)+2)
 if(len(context)>max_context_len):
print("data", idx, ":too long")
 continue
if unk_token in context:
print("data", idx, "tokenize with [UNK]", ":discard")
 continue
for idx, word in enumerate(context):
if model_name == "bert":
if word[0:2]=="
word = word[2:]
if model_name == "roberta":
if word[0]=="Ġ":
word = word[1:]
if model_name == "xlnet":
if word[0]=="▁":
word = word[1:]
if pt>=len(context_raw):
print(context_raw)
 print(context)
 print(idx, word)
 while is_whitespace(context_raw[pt]):
pt+=1
if pt>=len(context_raw):
print(context_raw)
 print(context)
 print(idx, word)
 if word == '"':
if context_raw[pt:pt+2] != "``" and context_raw[pt:pt+2] != "''":
print(context_raw)
 print(pt)
 print(context_raw[pt:pt+len(word)], word)
 char_to_word[pt:pt+2]= [idx]*len(word)
 word_to_char.append(pt)
 pt = pt + 2
else:
if context_raw[pt:pt+len(word)].lower() != word.lower() :
print(context_raw)
 print(pt)
 print(context_raw[pt:pt+len(word)], word)
 char_to_word[pt:pt+len(word)]= [idx]*len(word)
 word_to_char.append(pt)
 pt = pt + len(word)
 word_to_char.append(pt)
 if train:
answer_text = row['selected_text']
answer_text = "".join(map(character_filter, answer_text))
answer = tokenizer.tokenize(answer_text)
 raw_start = context_raw.find(answer_text)
 assert(raw_start>=0)
 answer_start= char_to_word[raw_start]
answer_end = char_to_word[raw_start + len(answer_text)- 1]

 assert(answer_end>=0 and answer_start>=0)
 answer_start += len(question)+2
answer_end += len(question)+2
else:
answer_start = -1
answer_end = -1
assert(len(word_to_char)== len(question)+ len(context)+ 3)
 word_to_char_list.append(word_to_char)
 example = data.Example.fromlist([
[cls_token]+question+[sep_token]+ context+[sep_token],
answer_start,
answer_end],
fields 
)
 examples.append(example)
 if train:
return data.Dataset(examples, fields)
 else:
return data.Dataset(examples, fields), context_raw_test, word_to_char_list<define_variables>
score_pca, pred_pca = create_data_n_train('PCA')
 score_mean, pred_mean = create_data_n_train('MEAN')
 print('PCA Score:', score_pca, ' Mean Encoding Score: ', score_mean )
<load_from_csv><EOS> 
submission_pca = pd.concat([df_test['PassengerId'], pd.Series(pred_pca, name="Survived")] ,axis=1)
 submission_pca.to_csv('submission_pca.csv', index=False)
 submission_mean = pd.concat([df_test['PassengerId'], pd.Series(pred_mean, name="Survived")] ,axis=1)
 submission_mean.to_csv('submission_mean.csv', index=False)
 display(submission_pca.head())
 display(submission_mean.head() )