Kaggle competition: Titanic - Machine Learning from Disaster
test2 = test.copy() 
<feature_engineering>
plt.style.available
test['Age']=test[['Age','Pclass']].apply(impute_age,axis = 1 )<drop_column>
train_df = pd.read_csv(".. /input/titanic/train.csv")
 test_df = pd.read_csv(".. /input/titanic/test.csv")
 test_PassengerId = test_df["PassengerId"]
test.drop('Cabin',inplace = True,axis =1 )<categorify>
category2= ["Cabin", "Name", "Ticket"]
for c in category2:
print(f"{train_df[c].value_counts() }
" )
sex = pd.get_dummies(test['Sex'],drop_first=True)
 embark = pd.get_dummies(test['Embarked'],drop_first=True )<concatenate>
train_df[["Pclass", "Survived"]].groupby(["Pclass"], as_index=False ).mean().sort_values(
 by="Survived", ascending=False )
test = pd.concat([test,sex,embark],axis=1 )<drop_column>
train_df[["Sex", "Survived"]].groupby(["Sex"], as_index=False ).mean().sort_values(
 by="Survived", ascending=False )
test.drop(['Sex','Embarked','PassengerId','Name','Ticket'],axis=1,inplace=True )<correct_missing_values>
train_df[["SibSp", "Survived"]].groupby(["SibSp"], as_index=False ).mean().sort_values(
 by="Survived", ascending=False )
test['Fare'].fillna(test['Fare'].mean() ,inplace=True )<categorify>
train_df[["Parch", "Survived"]].groupby(["Parch"], as_index=False ).mean().sort_values(
 by="Survived", ascending=False )
feature_scale = ['Age','Fare']
test[feature_scale] = st.fit_transform(test[feature_scale] )<import_modules>
train_df[["Sex", "Pclass", "Survived"]].groupby(["Sex", "Pclass"], as_index=False ).mean().sort_values(
 by="Survived", ascending=False )
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier 
<choose_model_class>
train_df[["Sex", "Pclass", "Parch", "SibSp","Survived"]].groupby(
 ["Sex", "Pclass", "Parch", "SibSp"], as_index=False ).mean().sort_values(by="Survived",
ascending=False)[:60]
level1 = LogisticRegression()
model = StackingClassifier(estimators=level0,final_estimator=level1,cv=5 )<train_model>
def detect_outliers(df, features):
outlier_indices=list()
for c in features:
Q1 = np.percentile(df[c], 25)
 Q3 = np.percentile(df[c], 75)
 IQR = Q3 - Q1
outlier_step = IQR*1.5
outlier_detect_column = df[(df[c]<(Q1-outlier_step)) |(df[c]>(Q3+outlier_step)) ].index
outlier_indices.extend(outlier_detect_column)
 outlier_indices = Counter(outlier_indices)
 multiple_outlier = list(i for i, j in outlier_indices.items() if j>2)
 return multiple_outlier
model.fit(x,y )<compute_test_metric>
train_df.loc[detect_outliers(train_df, ["Age", "SibSp", "Parch", "Fare"])]
model.score(x,y )<predict_on_test>
train_df = train_df.drop(detect_outliers(train_df, ["Age", "SibSp", "Parch", "Fare"]), axis=0 ).reset_index(
 drop = True )
y_predicted = model.predict(test )<create_dataframe>
train_df_len = len(train_df)
 train_df = pd.concat([train_df, test_df],axis=0 ).reset_index(drop=True )
submission = pd.DataFrame({
"PassengerId":test2['PassengerId'],
"Survived":y_predicted
} )<save_to_csv>
train_df.columns[train_df.isnull().any() ]
submission.to_csv('first_kaggale_titanic_submission.csv',index=False )<import_modules>
train_df.isnull().sum()
import numpy as np
import pandas as pd
import os
import tokenizers
import string
import torch
import transformers
import torch.nn as nn
from torch.nn import functional as F
from tqdm import tqdm
import re<define_variables>
train_df[train_df["Embarked"].isnull() ]
MAX_LEN = 128
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 16
EPOCHS = 5
ROBERTA_PATH = ".. /input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True
)<define_search_model>
train_df["Embarked"] = train_df["Embarked"].fillna("C" )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<string_transform>
train_df.columns[train_df.isnull().any() ]
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<define_variables>
train_df[train_df["Fare"].isnull() ]
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if len(original_tweet.split())< 2:
filtered_output = original_tweet
if sentiment_val != "neutral" and verbose == True:
if filtered_output.strip().lower() != target_string.strip().lower() :
print("********************************")
 print(f"Output= {filtered_output.strip() }")
 print(f"Target= {target_string.strip() }")
 print(f"Tweet= {original_tweet.strip() }")
 print("********************************")
 jac = 0
return jac, filtered_output<load_from_csv>
train_df["Fare"] = train_df["Fare"].fillna(np.mean(train_df[train_df["Pclass"]==3]["Fare"]))
train_df[train_df["Fare"].isnull() ]
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
pd.set_option('mode.chained_assignment', None)
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True<load_pretrained>
gender_submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv' )
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(".. /input/tweet-rob-model/model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load(".. /input/tweet-rob-model/model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load(".. /input/tweet-rob-model/model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load(".. /input/tweet-rob-model/model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load(".. /input/tweet-rob-model/model_4.bin"))
model5.eval()
model6 = TweetModel(conf=model_config)
 model6.to(device)
 model6.load_state_dict(torch.load(".. /input/tweet-rob-model/model_5.bin"))
model6.eval()
model7 = TweetModel(conf=model_config)
 model7.to(device)
 model7.load_state_dict(torch.load(".. /input/tweet-rob-model/model_6.bin"))
model7.eval()
model8 = TweetModel(conf=model_config)
 model8.to(device)
 model8.load_state_dict(torch.load(".. /input/tweet-rob-model/model_7.bin"))
model8.eval()<define_variables>
train_df.PassengerId[train_df.Cabin.notnull() ].count()
final_output = []<create_dataframe>
train_df.PassengerId[train_df.Age.notnull() ].count()
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start6, outputs_end6 = model6(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start7, outputs_end7 = model7(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start8, outputs_end8 = model8(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(outputs_start1 + outputs_start2 +
outputs_start3 + outputs_start4 +
outputs_start5 + outputs_start6 +
outputs_start7 + outputs_start8)/ 8
outputs_end =(outputs_end1 + outputs_end2 +
outputs_end3 + outputs_end4 +
outputs_end5 + outputs_end6 +
outputs_end7 + outputs_end8)/ 8
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<save_to_csv>
train_df.Age = train_df.Age.median()
train_df.Age
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.to_csv("submission.csv", index=False )<set_options>
train_df[train_df.Embarked.isnull() ]
tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 tf.config.optimizer.set_jit(True)
 tf.config.optimizer.set_experimental_options(
 {"auto_mixed_precision": True} )<load_from_csv>
MaxPassEmbarked = train_df.groupby('Embarked' ).count() ['PassengerId']
train_df.Embarked[train_df.Embarked.isnull() ] = MaxPassEmbarked[MaxPassEmbarked == MaxPassEmbarked.max() ].index[0]
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df.dropna(inplace=True)
 test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df.loc[:, "selected_text"] = test_df.text.values
submission_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 print("train shape =", train_df.shape)
 print("test shape =", test_df.shape)
 PATH = ".. /input/bert-base-uncased/"
MAX_SEQUENCE_LENGTH = 128
if PATH == ".. /input/bert-base-uncased/":
TOKENIZER = BertWordPieceTokenizer(f"{PATH}/vocab.txt", lowercase=True, add_special_tokens=False)
 else:
TOKENIZER = BertWordPieceTokenizer(".. /input/huggingfacetransformermodels/model_classes/roberta/roberta-base-tokenizer/vocab.json",lowercase=True, add_special_tokens=False)
 train_df.head(10 )<categorify>
train_df = train_df.drop(['PassengerId','Name','Ticket','Cabin'],axis=1)
 train_df
def preprocess(tweet, selected_text, sentiment):

 tweet = tweet.decode('utf-8')
 selected_text = selected_text.decode('utf-8')
 sentiment = sentiment.decode('utf-8')
 tweet = " ".join(str(tweet ).split())
 selected_text = " ".join(str(selected_text ).split())
 idx_start, idx_end = None, None
for index in(i for i, c in enumerate(tweet)if c == selected_text[0]):
if tweet[index:index+len(selected_text)] == selected_text:
idx_start = index
idx_end = index + len(selected_text)
 break
intersection = [0] * len(tweet)
 if idx_start != None and idx_end != None:
for char_idx in range(idx_start, idx_end):
intersection[char_idx] = 1
enc = TOKENIZER.encode(tweet)
 input_ids_orig, offsets = enc.ids, enc.offsets
target_idx = []
for i,(o1, o2)in enumerate(offsets):
if sum(intersection[o1: o2])> 0:
target_idx.append(i)
 target_start = target_idx[0]
target_end = target_idx[-1]
if PATH==".. /input/bert-base-uncased/":
sentiment_map = {
'positive': 3893,
'negative': 4997,
'neutral': 8699,
}
else:
sentiment_map = {
'positive': 3112,
'negative': 4366,
'neutral': 8796,
}
input_ids = [101] + [sentiment_map[sentiment]] + [102] + input_ids_orig + [102]
input_type_ids = [0, 0, 0] + [1] *(len(input_ids_orig)+ 1)
 attention_mask = [1] *(len(input_ids_orig)+ 4)
 offsets = [(0, 0),(0, 0),(0, 0)] + offsets + [(0, 0)]
target_start += 3
target_end += 3
padding_length = MAX_SEQUENCE_LENGTH - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 input_type_ids = input_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 elif padding_length < 0:
input_ids = input_ids[:padding_length-1] + [102]
attention_mask = attention_mask[:padding_length-1] + [1]
input_type_ids = input_type_ids[:padding_length-1] + [1]
offsets = offsets[:padding_length-1] + [(0, 0)]
if target_start >= MAX_SEQUENCE_LENGTH:
target_start = MAX_SEQUENCE_LENGTH - 1
if target_end >= MAX_SEQUENCE_LENGTH:
target_end = MAX_SEQUENCE_LENGTH - 1
return(
 input_ids, attention_mask, input_type_ids, offsets,
target_start, target_end, tweet, selected_text, sentiment, 
)
 class TweetSentimentDataset(tf.data.Dataset):
OUTPUT_TYPES =(
 tf.dtypes.int32, tf.dtypes.int32, tf.dtypes.int32,
tf.dtypes.int32, tf.dtypes.float32, tf.dtypes.float32,
tf.dtypes.string, tf.dtypes.string, tf.dtypes.string, 
)
 OUTPUT_SHAPES =(
(MAX_SEQUENCE_LENGTH,),(MAX_SEQUENCE_LENGTH,),(MAX_SEQUENCE_LENGTH,), 
(MAX_SEQUENCE_LENGTH, 2),() ,() , 
() ,() ,() , 
)
 def _generator(tweet, selected_text, sentiment):
for tw, st, se in zip(tweet, selected_text, sentiment):
yield preprocess(tw, st, se)
 def __new__(cls, tweet, selected_text, sentiment):
return tf.data.Dataset.from_generator(
 cls._generator,
output_types=cls.OUTPUT_TYPES,
output_shapes=cls.OUTPUT_SHAPES,
args=(tweet, selected_text, sentiment)
)
 @staticmethod
def create(dataframe, batch_size, shuffle_buffer_size=-1):
dataset = TweetSentimentDataset(
 dataframe.text.values,
dataframe.selected_text.values,
dataframe.sentiment.values 
)
 dataset = dataset.cache()
if shuffle_buffer_size != -1:
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset 
<choose_model_class>
train_df.columns[train_df.isnull().any() ]
class BertQAModel(TFBertPreTrainedModel):
DROPOUT_RATE = 0.5
NUM_HIDDEN_STATES = 2
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.bert = TFBertMainLayer(config, name="bert")
 self.concat = L.Concatenate()
self.dropout = L.Dropout(self.DROPOUT_RATE)
 self.hidden_output=L.Dense(
 128,
kernel_initializer=TruncatedNormal(stddev=config.initializer_range),
dtype='float32',
name='hidden_output')
 self.qa_outputs = L.Dense(
 config.num_labels,
kernel_initializer=TruncatedNormal(stddev=config.initializer_range),
dtype='float32',
name="qa_outputs")
 @tf.function
def call(self, inputs, **kwargs):
tjl, _, hidden_states = self.bert(inputs, **kwargs)
 hidden_states = self.dropout(tjl, training=kwargs.get("training", False))
hidden_ouputs= self.hidden_output(hidden_states)
 hidden_outputs=self.dropout(hidden_ouputs,training=kwargs.get("training", False))
logits = self.qa_outputs(hidden_outputs)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, axis=-1)
 end_logits = tf.squeeze(end_logits, axis=-1)
 return start_logits, end_logits
def train(model, dataset, loss_fn, optimizer):
@tf.function
def train_step(model, inputs, y_true, loss_fn, optimizer):
with tf.GradientTape() as tape:
y_pred = model(inputs, training=True)
 loss = loss_fn(y_true[0], y_pred[0])
 loss += loss_fn(y_true[1], y_pred[1])
 scaled_loss = optimizer.get_scaled_loss(loss)
 scaled_gradients = tape.gradient(scaled_loss, model.trainable_variables)
 gradients = optimizer.get_unscaled_gradients(scaled_gradients)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss, y_pred
epoch_loss = 0.
 for batch_num, sample in enumerate(dataset):
loss, y_pred = train_step(
 model, sample[:3], sample[4:6], loss_fn, optimizer)
 epoch_loss += loss
print(
 f"training...batch {batch_num+1:03d} : "
f"train loss {epoch_loss/(batch_num+1):.3f} ",
end='\r')
 def predict(model, dataset, loss_fn, optimizer):
@tf.function
def predict_step(model, inputs):
return model(inputs)
 def to_numpy(*args):
out = []
for arg in args:
if arg.dtype == tf.string:
arg = [s.decode('utf-8')for s in arg.numpy() ]
out.append(arg)
 else:
arg = arg.numpy()
out.append(arg)
 return out
offset = tf.zeros([0, MAX_SEQUENCE_LENGTH, 2], dtype=tf.dtypes.int32)
 text = tf.zeros([0,], dtype=tf.dtypes.string)
 selected_text = tf.zeros([0,], dtype=tf.dtypes.string)
 sentiment = tf.zeros([0,], dtype=tf.dtypes.string)
 pred_start = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 pred_end = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 for batch_num, sample in enumerate(dataset):
print(f"predicting...batch {batch_num+1:03d}"+" "*20, end='\r')
 y_pred = predict_step(model, sample[:3])
 pred_start = tf.concat(( pred_start, y_pred[0]), axis=0)
 pred_end = tf.concat(( pred_end, y_pred[1]), axis=0)
 offset = tf.concat(( offset, sample[3]), axis=0)
 text = tf.concat(( text, sample[6]), axis=0)
 selected_text = tf.concat(( selected_text, sample[7]), axis=0)
 sentiment = tf.concat(( sentiment, sample[8]), axis=0)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
to_numpy(pred_start, pred_end, text, selected_text, sentiment, offset)
 return pred_start, pred_end, text, selected_text, sentiment, offset
def decode_prediction(pred_start, pred_end, text, offset, sentiment):
def decode(pred_start, pred_end, text, offset):
decoded_text = ""
for i in range(pred_start, pred_end+1):
decoded_text += text[offset[i][0]:offset[i][1]]
if(i+1)< len(offset)and offset[i][1] < offset[i+1][0]:
decoded_text += " "
return decoded_text
decoded_predictions = []
for i in range(len(text)) :
if sentiment[i] == "neutral" or len(text[i].split())< 3:
decoded_text = text[i]
else:
idx_start = np.argmax(pred_start[i])
 idx_end = np.argmax(pred_end[i])
 if idx_start > idx_end:
idx_end = idx_start
decoded_text = str(decode(idx_start, idx_end, text[i], offset[i]))
if len(decoded_text)== 0:
decoded_text = text[i]
decoded_predictions.append(decoded_text)
 return decoded_predictions
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c)) 
<load_pretrained>
label = LabelEncoder()
dicts = {}
label.fit(train_df.Sex.drop_duplicates())
 dicts['Sex'] = list(label.classes_)
 train_df.Sex = label.transform(train_df.Sex)
 label.fit(train_df.Embarked.drop_duplicates())
 dicts['Embarked'] = list(label.classes_)
 train_df.Embarked = label.transform(train_df.Embarked)
 train_df
num_folds = 8
num_epochs = 3
batch_size = 32
learning_rate = 3e-5
optimizer = tf.keras.optimizers.Adam(learning_rate)
 optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, 'dynamic')
 if PATH == ".. /input/bert-base-uncased/":
config = BertConfig(output_hidden_states=True, num_labels=2)
 else:
config= BertConfig.from_json_file(".. /input/huggingfacetransformermodels/model_classes/roberta/roberta-base-tf2-model/config.json")
 print(type(config))
config.output_hidden_states=True
BertQAModel.DROPOUT_RATE = 0.5
BertQAModel.NUM_HIDDEN_STATES =2
model = BertQAModel.from_pretrained(PATH, config=config)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 kfold = model_selection.StratifiedKFold(
 n_splits=num_folds, shuffle=True, random_state=42)
 test_preds_start = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 test_preds_end = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 for fold_num,(train_idx, valid_idx)in enumerate(
 kfold.split(X=train_df.text, y=train_df.sentiment.values)) :
print("
fold %02d" %(fold_num+1))
train_dataset = TweetSentimentDataset.create(
 train_df.iloc[train_idx], batch_size, shuffle_buffer_size=2048)
 valid_dataset = TweetSentimentDataset.create(
 train_df.iloc[valid_idx], batch_size, shuffle_buffer_size=-1)
 test_dataset = TweetSentimentDataset.create(
 test_df, batch_size, shuffle_buffer_size=-1)
 best_score = float('-inf')
 for epoch_num in range(num_epochs):
print("
epoch %03d" %(epoch_num+1))
train(model, train_dataset, loss_fn, optimizer)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
predict(model, valid_dataset, loss_fn, optimizer)
 selected_text_pred = decode_prediction(
 pred_start, pred_end, text, offset, sentiment)
 jaccards = []
for i in range(len(selected_text)) :
jaccards.append(
 jaccard(selected_text[i], selected_text_pred[i]))
score = np.mean(jaccards)
 print(f"valid jaccard epoch {epoch_num+1:03d}: {score}"+" "*15)
 if score > best_score:
best_score = score
test_pred_start, test_pred_end, test_text, _, test_sentiment, test_offset = \
predict(model, test_dataset, loss_fn, optimizer)
 test_preds_start += test_pred_start
test_preds_end += test_pred_end
session = tf.compat.v1.get_default_session()
graph = tf.compat.v1.get_default_graph()
del session, graph, model
model = BertQAModel.from_pretrained(PATH, config=config)
 selected_text_pred = decode_prediction(
 test_preds_start, test_preds_end, test_text, test_offset, test_sentiment)
 submission_df.loc[:, 'selected_text'] = selected_text_pred
submission_df.to_csv("submission.csv", index=False )<import_modules>
test_df.Age[test_df.Age.isnull() ] = test_df.Age.mean()
test_df.Fare[test_df.Fare.isnull() ] = test_df.Fare.median()
MaxPassEmbarked = test_df.groupby('Embarked' ).count() ['PassengerId']
test_df.Embarked[test_df.Embarked.isnull() ] = MaxPassEmbarked[MaxPassEmbarked == MaxPassEmbarked.max() ].index[0]
result = pd.DataFrame(test_df.PassengerId)
 test_df = test_df.drop(['Name','Ticket','Cabin','PassengerId'],axis=1)
 label.fit(dicts['Sex'])
 test_df.Sex = label.transform(test_df.Sex)
 label.fit(dicts['Embarked'])
 test_df.Embarked = label.transform(test_df.Embarked )
import os
import torch
import pandas as pd
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn import metrics
import transformers
import tokenizers
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
from tqdm.autonotebook import tqdm
import utils<define_variables>
target = train_df.Survived
train_df = train_df.drop(['Survived'], axis=1)
 kfold = 5
itog_val = {}
class config:
MAX_LEN = 128
TRAIN_BATCH_SIZE = 64
VALID_BATCH_SIZE = 16
EPOCHS = 5
BERT_PATH = ".. /input/bert-base-uncased/"
MODEL_PATH = "model.bin"
TRAINING_FILE = ".. /input/tweet-train-folds/train_folds.csv"
TOKENIZER = tokenizers.BertWordPieceTokenizer(
 f"{BERT_PATH}/vocab.txt",
lowercase=True
)<define_variables>
ROCtrainTRN, ROCtestTRN, ROCtrainTRG, ROCtestTRG = train_test_split(train_df, target, test_size=0.25 )
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):

 len_st = len(selected_text)
 idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[0]):
if tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids[1:-1]
tweet_offsets = tok_tweet.offsets[1:-1]
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 3893,
'negative': 4997,
'neutral': 8699
}
input_ids = [101] + [sentiment_id[sentiment]] + [102] + input_ids_orig + [102]
token_type_ids = [0, 0, 0] + [1] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 3 + tweet_offsets + [(0, 0)]
targets_start += 3
targets_end += 3
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}<feature_engineering>
model_rfc = RandomForestClassifier(n_estimators = 80, max_features='auto', criterion='entropy',max_depth=4)
 model_knc = KNeighborsClassifier(n_neighbors = 18)
 model_lr = LogisticRegression(penalty='l2', tol=0.01)
 model_svc = svm.SVC()
class TweetDataset:

 def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<load_pretrained>
model_rfc.fit(train_df, target)
 result.insert(1,'Survived', model_rfc.predict(test_df))
result.to_csv('predictions.csv', index=False )