Kaggle competition: Titanic - Machine Learning from Disaster
class TweetModel(transformers.BertPreTrainedModel):

 def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.bert = transformers.BertModel.from_pretrained(config.BERT_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.bert(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from numpy import mean
from numpy import std
import string
import warnings
def loss_fn(start_logits, end_logits, start_positions, end_positions):

 loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss<choose_model_class>
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
def train_fn(data_loader, model, optimizer, device, scheduler=None):

 model.train()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg )<compute_test_metric>
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import OneHotEncoder, LabelEncoder, StandardScaler
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):

 if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):

 model.eval()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg<load_from_csv>
dfTrain = pd.read_csv(dirname+"/train.csv")
 dfTest = pd.read_csv(dirname+"/test.csv")
 dfGenderSubmission = pd.read_csv(dirname+"/gender_submission.csv" )
def run(fold):

 dfx = pd.read_csv(config.TRAINING_FILE)
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=2 
)
 device = torch.device("cuda")
 model_config = transformers.BertConfig.from_pretrained(config.BERT_PATH)
 model_config.output_hidden_states = True
model = TweetModel(conf=model_config)
 model.to(device)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 es = utils.EarlyStopping(patience=2, mode="max")
 print(f"Training is Starting for fold={fold}")
 for epoch in range(3):
train_fn(train_data_loader, model, optimizer, device, scheduler=scheduler)
 jaccard = eval_fn(valid_data_loader, model, device)
 print(f"Jaccard Score = {jaccard}")
 es(jaccard, model, model_path=f"model_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break<load_from_csv>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 dfMrg = concat_df(dfTrain,dfTest)
 dfMrg.isna().sum()
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
titleyouth = dfMrg[(dfMrg['Title'] == 'Miss')|(dfMrg['Title'] == 'Mr')]
dfMrg.loc[titleyouth[(titleyouth['Age'] < 21)].index.tolist() ,'Title'] = 'Youth'
device = torch.device("cuda")
 model_config = transformers.BertConfig.from_pretrained(config.BERT_PATH)
 model_config.output_hidden_states = True<load_pretrained>
titleDr = dfMrg[(dfMrg['Title'] == 'Dr')].dropna()
dfMrg.loc[titleDr[(titleDr['Sex'] == 'female')].index.tolist() ,'Title'] = 'Mrs'
dfMrg['Title'] = dfMrg['Title'].replace(['Dr'],'Mr')
 pd.crosstab(dfMrg['Title'],dfMrg['Sex'] )
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load("model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load("model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load("model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load("model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load("model_4.bin"))
model5.eval()<create_dataframe>
dfMrg.set_index('Title' ).isna().sum(level=0)['Age']
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(
 outputs_start1
+ outputs_start2
+ outputs_start3
+ outputs_start4
+ outputs_start5 
)/ 5
outputs_end =(
 outputs_end1
+ outputs_end2
+ outputs_end3
+ outputs_end4
+ outputs_end5 
)/ 5
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<string_transform>
age_by_pclass_sex = dfMrg.groupby(['Sex', 'Pclass'] ).median() ['Age']
def post_process(selected):
return " ".join(set(selected.lower().split()))<save_to_csv>
dfMrg['Age'] = dfMrg.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.selected_text = sample.selected_text.map(post_process)
 sample.to_csv("submission.csv", index=False )<import_modules>
dfMrg[dfMrg['Embarked'].isnull() ]
dfMrg['Embarked'] = dfMrg['Embarked'].fillna('S' )
import numpy as np
import pandas as pd
import json<load_from_csv>
dfMrg[dfMrg['Fare'].isnull() ]
med_fare = dfMrg.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
dfMrg['Fare'] = dfMrg['Fare'].fillna(med_fare )
pd_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 pd_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )<prepare_x_and_y>
dfMrg['Deck'] = dfMrg['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M' )
train = np.array(pd_train)
 test = np.array(pd_test )<find_best_params>
df_all_decks = dfMrg.groupby(['Deck', 'Pclass'] ).count().drop(columns=['Survived', 'Sex', 'Age', 'SibSp', 'Parch',
'Fare', 'Embarked', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name': 'Count'} ).transpose()
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1<define_variables>
def get_pclass_dist(df):
deck_counts = {'A': {}, 'B': {}, 'C': {}, 'D': {}, 'E': {}, 'F': {}, 'G': {}, 'M': {}, 'T': {}}
decks = df.columns.levels[0]
for deck in decks:
for pclass in range(1, 4):
try:
count = df[deck][pclass][0]
deck_counts[deck][pclass] = count
except KeyError:
deck_counts[deck][pclass] = 0
df_decks = pd.DataFrame(deck_counts)
 deck_percentages = {}
for col in df_decks.columns:
deck_percentages[col] = [(count / df_decks[col].sum())* 100 for count in df_decks[col]]
return deck_counts, deck_percentages
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in train:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/train.json', 'w')as outfile:
json.dump(output, outfile )<categorify>
all_deck_count, all_deck_per = get_pclass_dist(df_all_decks)
 display_pclass_dist(all_deck_per )
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in test:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/test.json', 'w')as outfile:
json.dump(output, outfile )<load_pretrained>
idx = dfMrg[dfMrg['Deck'] == 'T'].index
dfMrg.loc[idx, 'Deck'] = 'A'
!python /kaggle/input/pytorchtransformers/transformers-2.5.1/examples/run_squad.py \
--model_type roberta \
--model_name_or_path roberta-large \
--do_lower_case \
--do_train \
--do_eval \
--data_dir./data \
--cache_dir /kaggle/input/cached-roberta-large-pretrained/cache \
--train_file train.json \
--predict_file test.json \
--learning_rate 3e-5 \
--num_train_epochs 3 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir results_roberta_large \
--per_gpu_eval_batch_size=16 \
--per_gpu_train_batch_size=16 \
--save_steps=100000<string_transform>
df_all_decks_survived = dfMrg.groupby(['Deck', 'Survived'] ).count().drop(columns=['Sex', 'Age', 'SibSp', 'Parch', 'Fare',
'Embarked', 'Pclass', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
def f(selected):
return " ".join(set(selected.lower().split()))<load_from_csv>
def get_survived_dist(df):
surv_counts = {'A':{}, 'B':{}, 'C':{}, 'D':{}, 'E':{}, 'F':{}, 'G':{}, 'M':{}}
decks = df.columns.levels[0]
for deck in decks:
for survive in range(0, 2):
surv_counts[deck][survive] = df[deck][survive][0]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
return surv_counts, surv_percentages
def display_surv_dist(percentages):
df_survived_percentages = pd.DataFrame(percentages ).transpose()
deck_names =('A', 'B', 'C', 'D', 'E', 'F', 'G', 'M')
 bar_count = np.arange(len(deck_names))
bar_width = 0.85
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(20, 10))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Deck', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, deck_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage in Decks', size=18, y=1.05)
 plt.show()
all_surv_count, all_surv_per = get_survived_dist(df_all_decks_survived)
 display_surv_dist(all_surv_per )
predictions = json.load(open('results_roberta_large/predictions_.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if pd_test['sentiment'][i] == 'neutral':
submission.loc[i, 'selected_text'] = f(pd_test['text'][i])
 else:
submission.loc[i, 'selected_text'] = f(predictions[id_] )<save_to_csv>
dfMrg['Deck'] = dfMrg['Deck'].replace(['A', 'B', 'C'], 'ABC')
 dfMrg['Deck'] = dfMrg['Deck'].replace(['D', 'E'], 'DE')
 dfMrg['Deck'] = dfMrg['Deck'].replace(['F', 'G'], 'FG' )
submission.to_csv('submission.csv', index=False )<init_hyperparams>
dfMrg['Deck'].value_counts()
batch_size = 16
lr = 5e-5
epochs = 2
max_seq_len = 128
doc_stride = 64
cross_validation = True
K = 2
post_processing = True<import_modules>
dfMrg.drop(['Cabin'], inplace=True, axis=1 )
import numpy as np
import pandas as pd
import json
import os<load_from_csv>
df_train, df_test = divide_df(dfMrg)
 dfs = [df_train, df_test]
pd_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 pd_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )<prepare_x_and_y>
def display_missing(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in dfs:
display_missing(df )
np_train = np.array(pd_train)
 np_test = np.array(pd_test )<split>
dfMrg = concat_df(df_train, df_test)
 dfMrg.head()
def split_data(num_examples, K):
np.random.seed(0)
 idx = np.arange(num_examples)
 np.random.shuffle(idx)
 boundary = num_examples // K
splits = [{} for _ in range(K)]
for i in range(K):
splits[i]['valid_idx'] = idx[i*boundary:(i+1)*boundary]
splits[i]['train_idx'] = np.concatenate(( idx[:i*boundary], idx[(i+1)*boundary:]))
valid = np_train[splits[i]['valid_idx']]
d = {'neutral':0, 'positive':0, 'negative':0}
for line in valid:
d[line[-1]] += 1
print(d)
 return splits<split>
dfMrg['Fare'] = pd.qcut(dfMrg['Fare'], 13 )
splits = split_data(len(np_train), K )<categorify>
dfMrg['Age'] = pd.qcut(dfMrg['Age'],10 )
def convert_data(data, directory, filename):
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in data:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(context)!= str:
print(context, type(context))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer})
 qas.append({'question': '[MASK]', 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 if not os.path.exists(directory):
os.makedirs(directory)
 with open(os.path.join(directory, filename), 'w')as outfile:
json.dump(output, outfile )<categorify>
dfMrg['Ticket_Frequency'] = dfMrg.groupby('Ticket')['Ticket'].transform('count' )
for i, split in enumerate(splits):
data = np_train[split['train_idx']]
directory = 'split_' + str(i+1)
 filename = 'train.json'
convert_data(data, directory, filename )<categorify>
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
dfMrg['Family'] = extract_surname(dfMrg['Name'])
 df_train = dfMrg.loc[:890]
df_test = dfMrg.loc[891:]
dfs = [df_train, df_test]
data = np_train
directory = 'original'
filename = 'train.json'
convert_data(data, directory, filename)
 data = np_test
filename = 'test.json'
convert_data(data, directory, filename )<load_pretrained>
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA
def run_script(train_file, predict_file, batch_size=16, lr=5e-5, epochs=2, max_seq_len=128, doc_stride=64):
!python /kaggle/input/pytorchtransformers/transformers-2.5.1/examples/run_squad.py \
--model_type distilbert \
--model_name_or_path distilbert-base-uncased \
--cache_dir /kaggle/input/cached-distilbert-base-uncased/cache \
--do_lower_case \
--do_train \
--do_eval \
--train_file=$train_file \
--predict_file=$predict_file \
--overwrite_cache \
--learning_rate=$lr \
--num_train_epochs=$epochs \
--max_seq_length=$max_seq_len \
--doc_stride=$doc_stride \
--output_dir./results \
--overwrite_output_dir \
--per_gpu_eval_batch_size=$batch_size \
--per_gpu_train_batch_size=$batch_size \
--save_steps=100000<split>
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2
if cross_validation:
for i in range(1, K+1):
train_file = "split_" + str(i)+ "/train.json"
predict_file = "original/train.json"
run_script(train_file, predict_file, batch_size, lr, epochs, max_seq_len, doc_stride)
 !mv "results/predictions_.json" "results/predictions_"$i".json"<string_transform>
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<load_from_disk>
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )
def evaluate(splits, np_train, post_processing=False):
K = len(splits)
 predictions = [json.load(open('results/predictions_' + str(i+1)+ '.json', 'r')) for i in range(K)]
train_score = [{'neutral':[], 'positive':[], 'negative':[], 'total':[]} for _ in range(K+1)]
valid_score = [{'neutral':[], 'positive':[], 'negative':[], 'total':[]} for _ in range(K+1)]
for train_idx, line in enumerate(np_train):
text_id = line[0]
text = line[1]
answer = line[2]
sentiment = line[-1]
if type(text)!= str:
continue
for i, prediction in enumerate(predictions):
if text_id not in prediction:
print('key error:', text_id)
 continue
else:
if post_processing and(sentiment == 'neutral' or len(text.split())<= 0):
score = jaccard(answer, text)
 else:
score = jaccard(answer, prediction[text_id])
 if train_idx in splits[i]['valid_idx']:
valid_score[i][sentiment].append(score)
 valid_score[i]['total'].append(score)
 valid_score[K][sentiment].append(score)
 valid_score[K]['total'].append(score)
 else:
train_score[i][sentiment].append(score)
 train_score[i]['total'].append(score)
 train_score[K][sentiment].append(score)
 train_score[K]['total'].append(score)
 for i, score_dict in enumerate([train_score, valid_score]):
if i == 0:
print('train score
')
 else:
print('valid score
')
 for j in range(K+1):
for sentiment in ['neutral', 'positive', 'negative', 'total']:
score = np.array(score_dict[j][sentiment])
 if j < K:
print('split', j+1)
 else:
print('all data')
 print(sentiment + ' - ' + str(len(score)) + ' examples, average score: ' + str(score.mean()))
print()<compute_test_metric>
dfMrg = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
dfMrg.drop(columns=drop_cols, inplace=True)
 dfMrg.head()
if cross_validation:
evaluate(splits, np_train, post_processing )<split>
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
train_file = "original/train.json"
predict_file = "original/test.json"
run_script(train_file, predict_file, batch_size, lr, epochs, max_seq_len, doc_stride)
 !mv results/predictions_.json results/test_predictions.json<load_from_csv>
y_test = dfGenderSubmission.drop("PassengerId", axis=1 ).copy()
predictions = json.load(open('results/test_predictions.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if post_processing and(pd_test['sentiment'][i] == 'neutral' or len(pd_test['text'][i].split())<= 0):
submission.loc[i, 'selected_text'] = pd_test['text'][i]
else:
submission.loc[i, 'selected_text'] = predictions[id_]<save_to_csv>
SEED = 42
submission.to_csv('submission.csv', index=False )<set_options>
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1 )
tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 tf.config.optimizer.set_jit(True)
 tf.config.optimizer.set_experimental_options(
 {"auto_mixed_precision": True} )<load_from_csv>
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import StratifiedKFold
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df.dropna(inplace=True)
 test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df.loc[:, "selected_text"] = test_df.text.values
submission_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 print("train shape =", train_df.shape)
 print("test shape =", test_df.shape)
 PATH = ".. /input/bert-base-uncased/"
MAX_SEQUENCE_LENGTH = 128
TOKENIZER = BertWordPieceTokenizer(f"{PATH}/vocab.txt", lowercase=True, add_special_tokens=False)
 train_df.head(10 )<categorify>
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 importances = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=dfMrg.columns)
 fprs, tprs, scores = [], [], []
skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, y_train), 1):
print('Fold {}
'.format(fold))
leaderboard_model.fit(X_train[trn_idx], y_train[trn_idx])
 trn_fpr, trn_tpr, trn_thresholds = roc_curve(y_train[trn_idx], leaderboard_model.predict_proba(X_train[trn_idx])[:, 1])
 trn_auc_score = auc(trn_fpr, trn_tpr)
 val_fpr, val_tpr, val_thresholds = roc_curve(y_train[val_idx], leaderboard_model.predict_proba(X_train[val_idx])[:, 1])
 val_auc_score = auc(val_fpr, val_tpr)
 scores.append(( trn_auc_score, val_auc_score))
fprs.append(val_fpr)
 tprs.append(val_tpr)
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 1]
importances.iloc[:, fold - 1] = leaderboard_model.feature_importances_
oob += leaderboard_model.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, leaderboard_model.oob_score_))
print('Average OOB Score: {}'.format(oob))
<choose_model_class><EOS> 
class_survived = [col for col in probs.columns if col.endswith('Prob_1')]
probs['1'] = probs[class_survived].sum(axis=1)/ N
probs['0'] = probs.drop(columns=class_survived ).sum(axis=1)/ N
probs['pred'] = 0
pos = probs[probs['1'] >= 0.5].index
probs.loc[pos, 'pred'] = 1
y_pred = probs['pred'].astype(int)
 submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = y_pred.values
submission_df.to_csv('submissions.csv', header=True, index=False)
 submission_df.head(10 )