Kaggle competition: New York City Taxi Fare Prediction
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<feature_engineering>
train = pd.read_csv(".. /input/train.csv", nrows = 1000000)
 test = pd.read_csv(".. /input/test.csv" )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>
train.isnull().sum().sort_values(ascending=False )
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
test.isnull().sum().sort_values(ascending=False )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<init_hyperparams>
train = train.dropna()
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
Counter(train['fare_amount']<0 )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
train = train.drop(train[train['fare_amount']<0].index, axis=0)
 train.shape
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
train['fare_amount'].sort_values(ascending=False )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<set_options>
train[train['passenger_count']>6]
warnings.filterwarnings('ignore' )<set_options>
train = train.drop(train[train['passenger_count']==208].index,axis=0 )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<feature_engineering>
train[train['pickup_latitude']<-90]
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
train[train['pickup_latitude']>90]
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
train = train.drop(((train[train['pickup_latitude']<-90])|(train[train['pickup_latitude']>90])).index, axis=0 )
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
train[train['pickup_longitude']<-180]
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
train[train['pickup_longitude']>180]
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
train = train.drop(((train[train['pickup_longitude']<-180])|(train[train['pickup_longitude']>180])).index, axis=0 )
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )<load_from_csv>
train[train['dropoff_latitude']<-90]
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
train[train['dropoff_latitude']>90]
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
train = train.drop(((train[train['dropoff_latitude']<-90])|(train[train['dropoff_latitude']>90])).index, axis=0 )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
train[train['dropoff_latitude']<-180]|train[train['dropoff_latitude']>180]
import numpy as np
import pandas as pd
from xgboost import XGBClassifier
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier,BaggingClassifier,AdaBoostClassifier,ExtraTreesClassifier,GradientBoostingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression,SGDClassifier,RidgeClassifier,PassiveAggressiveClassifier,Perceptron
from sklearn.neural_network import MLPClassifier
import re
import optuna
from optuna.samplers import TPESampler
from lightgbm import LGBMClassifier
from mlens.ensemble import SuperLearner
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle<load_from_csv>
train['key'] = pd.to_datetime(train['key'])
 train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'] )
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 PassengerId = test['PassengerId']<count_missing_values>
test['key'] = pd.to_datetime(test['key'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'] )
train.isna().sum()<count_missing_values>
def haversine_distance(lat1, long1, lat2, long2):
data = [train, test]
for i in data:
R = 6371
phi1 = np.radians(i[lat1])
 phi2 = np.radians(i[lat2])
 delta_phi = np.radians(i[lat2]-i[lat1])
 delta_lambda = np.radians(i[long2]-i[long1])
 a = np.sin(delta_phi / 2.0)** 2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(R * c)
 i['H_Distance'] = d
return d
test.isna().sum()<feature_engineering>
haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude' )
df = pd.concat([train, test])
 df = df.reset_index(drop=True)
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['IsAlone'] = 0
df.loc[df['FamilySize'] == 1, 'IsAlone'] = 1
df['Fare'] = df['Fare'].fillna(df['Fare'].median())
 df['Has_Cabin'] = df["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 df['Title'] = df['Name'].apply(get_title)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df['Title'] = df['Title'].replace('Mlle', 'Miss')
 df['Title'] = df['Title'].replace('Ms', 'Miss')
 df['Title'] = df['Title'].replace('Mme', 'Mrs')
 df['Sex'] = df['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
df['Title'] = df['Title'].map(title_mapping)
 df.loc[ df['Fare'] <= 7.91, 'Fare'] = 0
df.loc[(df['Fare'] > 7.91)&(df['Fare'] <= 14.454), 'Fare'] = 1
df.loc[(df['Fare'] > 14.454)&(df['Fare'] <= 31), 'Fare'] = 2
df.loc[ df['Fare'] > 31, 'Fare'] = 3
df['Fare'] = df['Fare'].astype(int )<feature_engineering>
data = [train,test]
for i in data:
i['Year'] = i['pickup_datetime'].dt.year
i['Month'] = i['pickup_datetime'].dt.month
i['Date'] = i['pickup_datetime'].dt.day
i['Day of Week'] = i['pickup_datetime'].dt.dayofweek
i['Hour'] = i['pickup_datetime'].dt.hour
df['Title2'] = df['Name'].apply(get_title)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
df.replace({'Title2': mapping}, inplace=True)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
for title in titles:
age_to_impute = df.groupby('Title2')['Age'].median() [titles.index(title)]
df.loc[(df['Age'].isnull())&(df['Title2'] == title), 'Age'] = age_to_impute
df.drop('Title2', axis = 1, inplace = True )<feature_engineering>
train.sort_values(['H_Distance','fare_amount'], ascending=False )
df['Last_Name'] = df['Name'].apply(lambda x: str.split(x, ",")[0])
 DEFAULT_SURVIVAL_VALUE = 0.5
df['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in df[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
df.loc[df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
df.loc[df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
df.loc[df['Family_Survival']!=0.5].shape[0] )<groupby>
train.loc[(( train['pickup_latitude']==0)&(train['pickup_longitude']==0)) &(( train['dropoff_latitude']!=0)&(train['dropoff_longitude']!=0)) &(train['fare_amount']==0)]
for _, grp_df in df.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
df.loc[df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
df.loc[df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(df[df['Family_Survival']!=0.5].shape[0]))<feature_engineering>
train = train.drop(train.loc[(( train['pickup_latitude']==0)&(train['pickup_longitude']==0)) &(( train['dropoff_latitude']!=0)&(train['dropoff_longitude']!=0)) &(train['fare_amount']==0)].index, axis=0 )
df.loc[ df['Age'] <= 16, 'Age'] = 0
df.loc[(df['Age'] > 16)&(df['Age'] <= 32), 'Age'] = 1
df.loc[(df['Age'] > 32)&(df['Age'] <= 48), 'Age'] = 2
df.loc[(df['Age'] > 48)&(df['Age'] <= 64), 'Age'] = 3
df.loc[ df['Age'] > 64, 'Age'] = 4 ;
df = df.drop(['PassengerId','Cabin','Name','SibSp','Parch','Embarked','Ticket','Last_Name'], axis=1 )<count_missing_values>
test.loc[(( test['pickup_latitude']==0)&(test['pickup_longitude']==0)) &(( test['dropoff_latitude']!=0)&(test['dropoff_longitude']!=0)) ]
df.isna().sum()<drop_column>
train.loc[(( train['pickup_latitude']!=0)&(train['pickup_longitude']!=0)) &(( train['dropoff_latitude']==0)&(train['dropoff_longitude']==0)) &(train['fare_amount']==0)]
train = df[df['Survived'].notnull() ]
test = df[df['Survived'].isnull() ]
test = test.drop(['Survived'], axis=1 )<split>
train = train.drop(train.loc[(( train['pickup_latitude']!=0)&(train['pickup_longitude']!=0)) &(( train['dropoff_latitude']==0)&(train['dropoff_longitude']==0)) &(train['fare_amount']==0)].index, axis=0 )
x = train.copy()
y = x.pop('Survived')
 x_test = test.copy()
x = x.values
y = y.values
x_test = x_test.values
std_scaler = StandardScaler()
x = std_scaler.fit_transform(x)
 xf_test = std_scaler.transform(x_test)
 x, x_val, y, y_val = train_test_split(x, y,test_size=0.2, shuffle=False )<find_best_model_class>
test.loc[(( test['pickup_latitude']!=0)&(test['pickup_longitude']!=0)) &(( test['dropoff_latitude']==0)&(test['dropoff_longitude']==0)) ]
class Optimizer:
def __init__(self, metric, trials=30):
self.metric = metric
self.trials = trials
self.sampler = TPESampler()
def objective(self, trial):
model = create_model(trial)
 model.fit(x, y)
 preds = model.predict(x_val)
 if self.metric == 'acc':
return accuracy_score(y_val, preds)
 else:
return f1_score(y_val, preds)
 def optimize(self):
study = optuna.create_study(direction="maximize", sampler=self.sampler)
 study.optimize(self.objective, n_trials=self.trials)
 return study.best_params<choose_model_class>
high_distance = train.loc[(train['H_Distance']>200)&(train['fare_amount']!=0)]
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 n_estimators = trial.suggest_int("n_estimators", 2, 150)
 min_samples_leaf = trial.suggest_int("min_samples_leaf", 1, 10)
 model = RandomForestClassifier(
 min_samples_leaf=min_samples_leaf,
n_estimators=n_estimators,
max_depth=max_depth, 
)
 return model
optimizer = Optimizer('f1')
 rf_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 rf_acc_params = optimizer.optimize()
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 n_estimators = trial.suggest_int("n_estimators", 1, 150)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0000001, 1)
 gamma = trial.suggest_uniform('gamma', 0.0000001, 1)
 subsample = trial.suggest_uniform('subsample', 0.0001, 1.0)
 model = XGBClassifier(
 learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth=max_depth,
gamma=gamma,
subsample=subsample, 
)
 return model
optimizer = Optimizer('f1')
 xgb_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 xgb_acc_params = optimizer.optimize()
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 n_estimators = trial.suggest_int("n_estimators", 1, 150)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0000001, 1)
 num_leaves = trial.suggest_int("num_leaves", 2, 3000)
 min_child_samples = trial.suggest_int('min_child_samples', 3, 200)
 model = LGBMClassifier(
 learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth=max_depth,
num_leaves=num_leaves,
min_child_samples=min_child_samples, 
)
 return model
optimizer = Optimizer('f1')
 lgb_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 lgb_acc_params = optimizer.optimize()
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 min_samples_split = trial.suggest_int('min_samples_split', 2, 16)
 min_weight_fraction_leaf = trial.suggest_uniform('min_weight_fraction_leaf', 0.0, 0.5)
 min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10)
 model = DecisionTreeClassifier(
 min_samples_split=min_samples_split,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_depth=max_depth,
min_samples_leaf=min_samples_leaf, 
)
 return model
optimizer = Optimizer('f1')
 dt_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 dt_acc_params = optimizer.optimize()
def create_model(trial):
n_estimators = trial.suggest_int('n_estimators', 2, 200)
 max_samples = trial.suggest_int('max_samples', 1, 100)
 model = BaggingClassifier(
 n_estimators=n_estimators,
max_samples=max_samples, 
)
 return model
optimizer = Optimizer('f1')
 bc_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 bc_acc_params = optimizer.optimize()
def create_model(trial):
n_neighbors = trial.suggest_int("n_neighbors", 2, 25)
 model = KNeighborsClassifier(n_neighbors=n_neighbors)
 return model
optimizer = Optimizer('f1')
 knn_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 knn_acc_params = optimizer.optimize()
def create_model(trial):
n_estimators = trial.suggest_int("n_estimators", 2, 150)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0005, 1.0)
 model = AdaBoostClassifier(
 n_estimators=n_estimators,
learning_rate=learning_rate, 
)
 return model
optimizer = Optimizer('f1')
 abc_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 abc_acc_params = optimizer.optimize()
def create_model(trial):
n_estimators = trial.suggest_int("n_estimators", 2, 150)
 max_depth = trial.suggest_int("max_depth", 2, 6)
 model = ExtraTreesClassifier(
 n_estimators=n_estimators,
max_depth=max_depth,
random_state=0 
)
 return model
optimizer = Optimizer('f1')
 et_f1_params = optimizer.optimize()
optimizer = Optimizer('acc')
 et_acc_params = optimizer.optimize() 
<choose_model_class>
high_distance['H_Distance'] = high_distance.apply(
 lambda row:(row['fare_amount'] - 2.50)/1.56,
axis=1
)
mdict = {
'RF': RandomForestClassifier() ,
'XGB': XGBClassifier() ,
'LGBM': LGBMClassifier() ,
'DT': DecisionTreeClassifier() ,
'KNN': KNeighborsClassifier() ,
'BC': BaggingClassifier() ,
'OARF': RandomForestClassifier(**rf_acc_params),
'OFRF': RandomForestClassifier(**rf_f1_params),
'OAXGB': XGBClassifier(**xgb_acc_params),
'OFXGB': XGBClassifier(**xgb_f1_params),
'OALGBM': LGBMClassifier(**lgb_acc_params),
'OFLGBM': LGBMClassifier(**lgb_f1_params),
'OADT': DecisionTreeClassifier(**dt_acc_params),
'OFDT': DecisionTreeClassifier(**dt_f1_params),
'OAKNN': KNeighborsClassifier(**knn_acc_params),
'OFKNN': KNeighborsClassifier(**knn_f1_params),
'OABC': BaggingClassifier(**bc_acc_params),
'OFBC': BaggingClassifier(**bc_f1_params),
'OAABC': AdaBoostClassifier(**abc_acc_params),
'OFABC': AdaBoostClassifier(**abc_f1_params),
'OAET': ExtraTreesClassifier(**et_acc_params),
'OFET': ExtraTreesClassifier(**et_f1_params),
'LR': LogisticRegression(max_iter=1000),
'ABC': AdaBoostClassifier() ,
'SGD': SGDClassifier() ,
'ET': ExtraTreesClassifier() ,
'GB': GradientBoostingClassifier() ,
'RDG': RidgeClassifier() ,
'PCP': Perceptron(max_iter=1500),
'PAC': PassiveAggressiveClassifier()
}<define_variables>
train.update(high_distance )
def create_model(trial):
model_names = list()
models_list = [
'RF', 'XGB', 'LGBM', 'DT',
'KNN', 'BC', 'OARF', 'OFRF',
'OAXGB', 'OFXGB', 'OALGBM',
'OFLGBM', 'OADT', 'OFDT',
'OAKNN', 'OFKNN', 'OABC',
'OFBC', 'OAABC', 'OFABC',
'OAET', 'OFET', 'LR',
'ABC', 'SGD', 'ET',
'GB', 'RDG',
'PCP', 'PAC'
]
head_list = [
'RF',
'XGB',
'LGBM',
'DT',
'KNN',
'BC',
'LR',
'ABC',
'SGD',
'ET',
'GB',
'RDG',
'PCP',
'PAC'
]
n_models = trial.suggest_int("n_models", 2, 6)
 for i in range(n_models):
model_item = trial.suggest_categorical('model_{}'.format(i), models_list)
 if model_item not in model_names:
model_names.append(model_item)
 folds = trial.suggest_int("folds", 2, 6)
 model = SuperLearner(
 folds=folds, 
)
 models = [
mdict[item] for item in model_names
]
model.add(models)
 head = trial.suggest_categorical('head', head_list)
 model.add_meta(
 mdict[head] 
)
 return model
def objective(trial):
model = create_model(trial)
 model.fit(x, y)
 preds = model.predict(x_val)
 score = accuracy_score(y_val, preds)
 return score
study = optuna.create_study(
 direction="maximize",
sampler= TPESampler() 
)
 study.optimize(
 objective,
n_trials=50
)<normalization>
train[train['H_Distance']==0]
model = SuperLearner(
 folds=folds, 
)
 models = [
mdict[item] for item in result
]
model.add(models)
 model.add_meta(mdict[head])
 xf = train.copy()
yf = xf.pop('Survived')
 xf = xf.values
xf = std_scaler.fit_transform(xf)
 yf = yf.values
model.fit(xf, yf )<save_to_csv>
train[(train['H_Distance']==0)&(train['fare_amount']==0)]
preds = model.predict(xf_test ).astype(int)
 output = pd.DataFrame({ 'PassengerId': PassengerId,
'Survived': preds })
 output.to_csv('boosted_tree.csv', index=False)
<load_from_csv>
train = train.drop(train[(train['H_Distance']==0)&(train['fare_amount']==0)].index, axis = 0 )
train_data_raw = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data_raw.head()<drop_column>
rush_hour = train.loc[(((train['Hour']>=6)&(train['Hour']<=20)) &(( train['Day of Week']>=1)&(train['Day of Week']<=5)) &(train['H_Distance']==0)&(train['fare_amount'] < 2.5)) ]
rush_hour
train_data=train_data_raw.drop(columns=['PassengerId','Name','Cabin','Ticket'])
 train_data.head()<filter>
train=train.drop(rush_hour.index, axis=0 )
train_data=train_data.loc[pd.notna(train_data.Embarked)]
train_data<categorify>
non_rush_hour = train.loc[(((train['Hour']<6)|(train['Hour']>20)) &(( train['Day of Week']>=1)&(train['Day of Week']<=5)) &(train['H_Distance']==0)&(train['fare_amount'] < 3.0)) ]
non_rush_hour
age_arr=train_data.Age.values
bool_arr=pd.isna(train_data.Age.values)
 total_age=0
num_age=0
for i in range(len(age_arr)) :
if bool_arr[i]==False:
total_age+=age_arr[i]
num_age+=1
avg_age=(total_age/num_age)
 for i in range(len(age_arr)) :
if bool_arr[i]==True:
age_arr[i]=avg_age
train_data.replace(to_replace=train_data.Age.values,value=age_arr)
 print(train_data.Age.values )<categorify>
weekends = train.loc[(( train['Day of Week']==0)|(train['Day of Week']==6)) &(train['H_Distance']==0)&(train['fare_amount'] < 3.0)]
weekends
scaled_age_arr=[round(age/avg_age,2)for age in age_arr]
train_data.Age=train_data.Age.replace(to_replace=train_data.Age.values,value=scaled_age_arr)
 fare_arr=train_data.Fare.values
total_fare=0
num_fare=len(fare_arr)
 for fare in fare_arr:
total_fare+=fare
avg_fare=total_fare/num_fare
scaled_fare_arr=[round(fare/avg_fare,2)for fare in fare_arr]
train_data.Fare=train_data.Fare.replace(to_replace=train_data.Fare.values,value=scaled_fare_arr)
 train_data<categorify>
train.loc[(train['H_Distance']!=0)&(train['fare_amount']==0)]
d_Sex={'male':0,'female':1}
d_Embarked={'S':0,'C':1,'Q':2}
train_data.Sex = train_data.Sex.replace(d_Sex)
 train_data.Embarked = train_data.Embarked.replace(d_Embarked)
 train_data.Embarked=train_data.Embarked.astype(int)
 train_data<split>
scenario_3 = train.loc[(train['H_Distance']!=0)&(train['fare_amount']==0)]
input_data=train_data[['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']][:]
target=train_data['Survived'][:]
print(input_data.values)
<randomize_order>
scenario_3.sort_values('H_Distance', ascending=False )
shuffled_indices = np.arange(len(input_data.values)-1)
 np.random.shuffle(shuffled_indices)
 shuffled_inputs = input_data.values[shuffled_indices]
shuffled_targets = target.values[shuffled_indices]
shuffled_inputs 
<split>
scenario_3['fare_amount'] = scenario_3.apply(
 lambda row:(( row['H_Distance'] * 1.56)+ 2.50), axis=1
)
num_train_samples=int(0.8*len(shuffled_inputs))
num_validation_samples=int(len(shuffled_inputs)) -num_train_samples
train_input=shuffled_inputs[:num_train_samples]
train_target=shuffled_targets[:num_train_samples]
validation_input=shuffled_inputs[num_train_samples:]
validation_target=shuffled_targets[num_train_samples:] 
<choose_model_class>
train.loc[(train['H_Distance']==0)&(train['fare_amount']!=0)]
input_size=7
output_size=2
hidden_layer_size=2700
num_hidden_layers=10
model=tf.keras.Sequential()
for i in range(num_hidden_layers):
model.add(tf.keras.layers.Dense(units=hidden_layer_size,activation='relu'))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(units=output_size,activation='softmax'))
learning_rate=0.005
optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate, beta_1=0.9)
 model.compile(optimizer=optimizer,loss=tf.keras.losses.SparseCategoricalCrossentropy() ,metrics=['accuracy'])
 batch_size=90
epochs=30
early_stopping=tf.keras.callbacks.EarlyStopping(monitor='val_loss',patience=10)
 def scheduler1(epoch, lr):
if epoch < 10:
return lr
elif epoch < 20:
return 0.003
elif epoch < 30:
return 0.002
elif epoch < 50:
return 0.001
else:
return 0.0009
def scheduler2(epoch, lr):
if epoch < 10:
return lr
else:
return lr * tf.math.exp(-0.1)
 def scheduler3(epoch, lr):
if epoch < 10:
return lr
elif epoch < 20:
return lr * tf.math.exp(-0.1)
 else:
return lr * tf.math.exp(-1.0)
 scheduler=scheduler3
lrs=tf.keras.callbacks.LearningRateScheduler(scheduler)
 model.fit(
 x=train_input,
y=train_target,
batch_size=batch_size,
epochs=epochs,
verbose=1,
callbacks=[
lrs
],
validation_data=(validation_input,validation_target)
)
<load_from_csv>
scenario_4 = train.loc[(train['H_Distance']==0)&(train['fare_amount']!=0)]
test_data_raw = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data=test_data_raw.drop(['Name','Ticket','Cabin'],axis='columns')
 test_data.head() 
<define_variables>
scenario_4.loc[(scenario_4['fare_amount']<=3.0)&(scenario_4['H_Distance']==0)]
age_arr=test_data.Age.values
bool_arr=pd.isna(test_data.Age.values)
 total_age=0
num_age=0
for i in range(len(age_arr)) :
if bool_arr[i]==False:
total_age+=age_arr[i]
num_age+=1
avg_age=(total_age/num_age)
 for i in range(len(age_arr)) :
if bool_arr[i]==True:
age_arr[i]=avg_age
scaled_age_arr=[round(age/avg_age,2)for age in age_arr]
test_data.Age=test_data.Age.replace(to_replace=test_data.Age.values,value=scaled_age_arr)
 fare_arr=test_data.Fare.values
bool_arr=pd.isna(test_data.Fare.values)
 total_fare=0
num_fare=0
for i in range(len(fare_arr)) :
if bool_arr[i]==False:
total_fare+=fare_arr[i]
num_fare+=1
avg_fare=(total_fare/num_fare)
 for i in range(len(fare_arr)) :
if bool_arr[i]==True:
fare_arr[i]=avg_fare
scaled_fare_arr=[round(fare/avg_fare,2)for fare in fare_arr]
test_data.Fare=test_data.Fare.replace(to_replace=test_data.Fare.values,value=scaled_fare_arr)
 test_data 
<categorify>
scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]
d_Sex={'male':0,'female':1}
d_Embarked={'S':0,'C':1,'Q':2}
test_data.Sex = test_data.Sex.replace(d_Sex)
 test_data.Embarked = test_data.Embarked.replace(d_Embarked)
 test_data.Embarked=test_data.Embarked.astype(int)
<drop_column>
scenario_4_sub = scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]
test_data.drop(['PassengerId'],axis=1 ).values<predict_on_test>
scenario_4_sub['H_Distance'] = scenario_4_sub.apply(
 lambda row:(( row['fare_amount']-2.50)/1.56), axis=1
)
probabilities = model.predict(test_data.drop(['PassengerId'],axis=1 ).values)
 predictions=list()
for p in probabilities:
if p[0]>p[1]:
predictions.append(0)
 else:
predictions.append(1)
 print(predictions )<save_to_csv>
train = train.drop(['key','pickup_datetime'], axis = 1)
 test = test.drop(['key','pickup_datetime'], axis = 1 )
output = pd.DataFrame({'PassengerId': test_data.PassengerId.values, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
x_train = train.iloc[:,train.columns!='fare_amount']
y_train = train['fare_amount'].values
x_test = test
with open('my_submission.csv','r')as f:
reader=csv.reader(f)
 for row in reader:
print(row )<set_options>
rf = RandomForestRegressor()
rf.fit(x_train, y_train)
 rf_predict = rf.predict(x_test)
warnings.filterwarnings("ignore")
<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = rf_predict
submission.to_csv('submission_1.csv', index=False)
 submission.head(20 )