Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
titanic = pd.read_csv("/kaggle/input/titanic/train.csv", sep=",")
 titanic_sub = pd.read_csv("/kaggle/input/titanic/test.csv", sep="," )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<compute_test_metric>
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
 for train_index, test_index in split.split(titanic, titanic['Sex']):
train_set = titanic.loc[train_index]
test_set = titanic.loc[test_index]
train_set = train_set.reset_index(drop=True )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<define_variables>
np.nanmean(train_set['Age'].loc[Title[Title == 'Miss'].index] )
embed_size = 300
max_features = 95000
maxlen = 70<define_variables>
np.nanmean(train_set['Age'].loc[Title[Title == 'Mrs'].index] )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<feature_engineering>
np.corrcoef(Family_members, People_on_ticket )
train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
x_train = train_df["question_text"].fillna("_
x_test = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_test = pad_sequences(x_test, maxlen=maxlen)
 y_train = train_df['target'].values<statistical_test>
sum(People_on_ticket<Family_members+1 )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
sum(People_on_ticket>Family_members+1 )
seed_everything()
glove_embeddings = load_glove(tokenizer.word_index)
 paragram_embeddings = load_para(tokenizer.word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 np.shape(embedding_matrix )<split>
print("Average age of lone passenger: ", round(np.nanmean(train_set_lone['Age']),0), sep="")
 print("Number of missing Age values: ", sum(np.isnan(train_set_lone['Age'])) , " - ",
round(( sum(np.isnan(train_set_lone['Age'])) *100/len(train_set_lone['Age'])) ,2), "% of train_set_lone.", sep="")
 print("The number of missing values is ",
round(sum(np.isnan(train_set_lone['Age'])) *100/sum(np.isnan(train_set['Age'])) ,2),
"% of entire missing values in Age variable in train_set.", sep="" )
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=10 ).split(x_train, y_train))<normalization>
outlier_ind = train_set.loc[train_set['Fare']==max(train_set['Fare'])].index
train_set = train_set.drop(outlier_ind)
 train_set = train_set.reset_index(drop=True )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
print("Minimum price for ticket in first class: ", min(train_set.loc[train_set['Pclass']==1]['Fare']), sep="" )
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 40
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(320, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<define_variables>
first_class = train_set.loc[train_set['Pclass']==1]
second_class = train_set.loc[train_set['Pclass']==2]
third_class = train_set.loc[train_set['Pclass']==3]
p_class = list(train_set['Pclass'])
 fare = list(train_set['Fare'])
 Fare_class = list()
for i in range(0,len(p_class)) :
if(p_class[i] == 1):
if(fare[i] < statistics.mean(second_class['Fare'])) :
Fare_class.append("cheap")
 else:
Fare_class.append("normal")
 elif(p_class[i] == 2):
if(fare[i] < statistics.mean(third_class['Fare'])) :
Fare_class.append("cheap")
 elif(fare[i] > statistics.mean(first_class['Fare'])) :
Fare_class.append("expensive")
 else:
Fare_class.append("normal")
 else:
if(fare[i] > statistics.mean(second_class['Fare'])) :
Fare_class.append("expensive")
 else:
Fare_class.append("normal")
 Fare_class = pd.Series(Fare_class)
 sns.countplot(Fare_class, hue='Survived', data=train_set)
 plt.legend(['No','Yes'], title="Survived", loc="upper right")
 plt.title("Fare_class: Survived vs Dead")
 plt.show()
batch_size = 512
n_epochs = 6<choose_model_class>
train_set_no_cabins = train_set.loc[np.where(pd.isnull(train_set['Cabin'])) ]
train_set_cabins = train_set.loc[~train_set.index.isin(train_set_no_cabins.index)]
train_set_cabins['Pclass'].value_counts()
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<compute_test_metric>
print("Average age of Southampton passenger:", round(np.nanmean(train_set[train_set['Embarked'] == 'S']['Age'])))
 print("Average age of Queenstown passenger:", round(np.nanmean(train_set[train_set['Embarked'] == 'Q']['Age'])))
 print("Average age of Cherbourg passenger:", round(np.nanmean(train_set[train_set['Embarked'] == 'C']['Age'])) )
def f1_smart(y_true, y_pred):
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_true,(y_pred > thresh ).astype(int))
thresholds.append([thresh, res])
 thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
best_f1 = thresholds[0][1]
return best_f1, best_thresh
def f1_smart_torch(y_true, y_pred):
y_true = y_true.cpu().data.numpy()
return f1_smart(y_true, y_pred)
 def save(m, info):
torch.save(info, 'best_model.info')
 torch.save(m, 'best_model.m')
 def load() :
m = torch.load('best_model.m')
 info = torch.load('best_model.info')
 return m, info<define_variables>
train_set_known_age = train_set_age[~np.isnan(train_set_age['Age'])]
train_set_known_age = train_set_known_age[(( train_set_known_age['Pclass']==3)& 
(train_set_known_age['Is_alone']==1)) |(train_set_known_age['Pclass']==3)]
print("Percentage of single and/or third class passengers with a known age: ",
round(len(train_set_known_age.index)*100/len(train_set.index), 2), "%", sep="" )
train_preds = np.zeros(( len(train_df)))
 test_preds = np.zeros(( len(test_df)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 best_thresholds = []
best_f1 = []
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 base_lr, max_lr = 0.0001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=max_lr)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 min_val_loss = 1000000000
step_size=300
best_thresh = 0
max_f1 = 0
for epoch in range(n_epochs):
start_time = time.time()
scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 model.train()
avg_loss = 0.
 step = 0
for x_batch, y_batch in tqdm(train_loader, disable=True):
if scheduler:
scheduler.batch_step()
step += 1
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_df)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
val_f1, threshold = f1_smart_torch(y_val_fold, valid_preds_fold)
 print('Optimal F1: {} at threshold: {}'.format(val_f1, threshold))
if avg_val_loss < min_val_loss:
print("Saving model with minimum Loss")
 save(m=model, info={'epoch': epoch, 'val_loss': avg_val_loss, 'val_f1': val_f1})
 min_val_loss = avg_val_loss
best_thresh = threshold
max_f1 = val_f1
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
best_thresholds.append(best_thresh)
 best_f1.append(max_f1)
 model,_ = load()
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder
from sklearn.pipeline import FeatureUnion
from sklearn.preprocessing import StandardScaler
search_result = threshold_search(y_train, train_preds)
 search_result<save_to_csv>
class TitleSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def get_title(self, obj):
title =(( obj.rsplit(',', 1)[1] ).rsplit('.', 1)[0] ).strip()
return title
def transform(self, X):
X.loc[:, 'Title'] = X[self._attribute_names].apply(self.get_title)
 X = X.drop(self._attribute_names, 1)
 return X.values
class TitleCoder(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_title(self, obj):
if(obj == 'Mr'):
title_c = 'Mr'
elif(obj == 'Mrs'):
title_c = 'Mrs'
elif(obj == 'Miss'):
title_c = 'Miss'
elif(obj == 'Master'):
title_c = 'Master'
else:
title_c = 'Other'
return title_c
def transform(self, X):
X.loc[:, 'Title_c'] = X[self._attribute_names].apply(self.code_title)
 X = X.drop(self._attribute_names, 1)
 return X.values
class AgeCoder(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_age(self, obj):
if(obj <= 15):
age_c = 'young'
elif(obj < 50):
age_c = 'mid'
elif(obj >= 50):
age_c = 'old'
else:
age_c = 'unknown'
return age_c
def transform(self, X):
X.loc[:, 'Age_c'] = X[self._attribute_names].apply(self.code_age)
 X = X.drop(self._attribute_names, 1)
 return X.values
class SibSpCoder(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_SibSp(self, obj):
if(obj == 0):
SibSp_c = 'zero'
elif(obj == 1):
SibSp_c = 'one'
else:
SibSp_c = 'more'
return SibSp_c
def transform(self, X):
X.loc[:, 'SibSp_c'] = X[self._attribute_names].apply(self.code_SibSp)
 X = X.drop(self._attribute_names, 1)
 return X.values
class ParchCoder(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_Parch(self, obj):
if(obj == 0):
Parch_c = 'zero'
elif(obj == 1 or obj == 2):
Parch_c = 'one/two'
else:
Parch_c = 'more'
return Parch_c
def transform(self, X):
X.loc[:, 'Parch_c'] = X[self._attribute_names].apply(self.code_Parch)
 X = X.drop(self._attribute_names, 1)
 return X.values
class Family_members(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
X.loc[:, 'Family_members'] = X[self._attribute_names] + X['Parch']
return X.values
class Family_membersCoder(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_Family_members(self, obj):
if(obj == 0):
Family_members_c = 'zero'
elif(obj == 1 or obj == 2):
Family_members_c = 'one/two'
elif(obj == 3):
Family_members_c = 'three'
else:
Family_members_c = 'more'
return Family_members_c
def transform(self, X):
X.loc[:, 'Family_members_c'] = X[self._attribute_names].apply(self.code_Family_members)
 X = X.drop(self._attribute_names, 1)
 return X.values
class People_on_ticket(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
counts = list()
tickets = list(X[self._attribute_names])
 for i in tickets:
counts.append(ticket_base.count(i))
X.loc[:, 'People_on_ticket'] = counts
return X.values
class People_on_ticketCoder(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_People_on_ticket(self, obj):
if(obj == 1):
People_on_ticket_c = 'one'
elif(obj == 2):
People_on_ticket_c = 'two'
elif(obj == 3):
People_on_ticket_c = 'three'
elif(obj == 4):
People_on_ticket_c = 'four'
else:
People_on_ticket_c = 'more'
return People_on_ticket_c
def transform(self, X):
X.loc[:, 'People_on_ticket_c'] = X[self._attribute_names].apply(self.code_People_on_ticket)
 X = X.drop(self._attribute_names, 1)
 return X.values
class Is_Alone(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
family_members = list(X['Family_members'])
 ppl_on_ticket = list(X[self._attribute_names])
 is_alone = list()
for i in range(0,len(ppl_on_ticket)) :
if(ppl_on_ticket[i] == 1 and family_members[i] == 0):
is_alone.append(1)
 else:
is_alone.append(0)
 X.loc[:, 'Is_alone'] = is_alone
return X.values
class Is_Cabin(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def code_Cabin(self, obj):
if(pd.isnull(obj)) :
is_cabin = 0
else:
is_cabin = 1
return is_cabin
def transform(self, X):
X.loc[:, 'Is_cabin'] = X[self._attribute_names].apply(self.code_Cabin)
 X = X.drop(self._attribute_names, 1)
 return X.values
class FareClass(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self._attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
first_class = X.loc[X['Pclass']==1]
second_class = X.loc[X['Pclass']==2]
third_class = X.loc[X['Pclass']==3]
p_class = list(X['Pclass'])
 fare = list(X[self._attribute_names])
 fare_class = list()
for i in range(0,len(p_class)) :
if(p_class[i] == 1):
if(fare[i] < statistics.mean(second_class['Fare'])) :
fare_class.append("cheap")
 else:
fare_class.append("normal")
 elif(p_class[i] == 2):
if(fare[i] < statistics.mean(third_class['Fare'])) :
fare_class.append("cheap")
 elif(fare[i] > statistics.mean(first_class['Fare'])) :
fare_class.append("expensive")
 else:
fare_class.append("normal")
 else:
if(fare[i] > statistics.mean(second_class['Fare'])) :
fare_class.append("expensive")
 else:
fare_class.append("normal")
 X.loc[:, 'Fare_class'] = fare_class
return X.values
submission = test_df[['qid']].copy()
submission['prediction'] = test_preds > search_result['threshold']
submission.to_csv('submission.csv', index=False )<set_options>
name = 'Name'
name_pipeline = Pipeline(steps=[ 
('get_title', TitleSelector(name))
])
 title = 'Title'
title_pipeline = Pipeline(steps=[ 
('code_title', TitleCoder(title))
])
 age = 'Age'
age_pipeline = Pipeline(steps=[ 
('code_age', AgeCoder(age))
])
 sibsp = 'SibSp'
sibsp_pipeline = Pipeline(steps=[ 
('code_sibsp', SibSpCoder(sibsp))
])
 sibsp_pipeline2 = Pipeline(steps=[ 
('create_family_members', Family_members(sibsp))
])
 parch = 'Parch'
parch_pipeline = Pipeline(steps=[ 
('code_parch', ParchCoder(parch))
])
 family_members = 'Family_members'
family_members_pipeline = Pipeline(steps=[ 
('code_family_members', Family_membersCoder(family_members))
])
 ticket = 'Ticket'
ticket_pipeline = Pipeline(steps=[ 
('create_People_on_ticket', People_on_ticket(ticket)) ,
])
 people_on_ticket = 'People_on_ticket'
ticket_pipeline2 = Pipeline(steps=[ 
('code_People_on_ticket', People_on_ticketCoder(people_on_ticket)) ,
])
 is_alone_pipeline = Pipeline(steps=[ 
('create_is_alone', Is_Alone(people_on_ticket)) ,
])
 cabin = 'Cabin'
cabin_pipeline = Pipeline(steps=[ 
('code_cabin', Is_Cabin(cabin)) ,
])
 fare = 'Fare'
fare_pipeline = Pipeline(steps=[ 
('fare_class', FareClass(fare)) ,
])
 attribs1 = ['Pclass', 'Title_c', 'Sex', 'Age_c',
'Family_members_c', 'Embarked',
]
attribs1_pipeline = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy="most_frequent")) , 
('encoder', OneHotEncoder(sparse=False)) , 
('standarizer', StandardScaler())
 ])
 attribs2 = ['SibSp', 'Parch',
'Fare', 'Is_cabin']
attribs2_pipeline = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy="median")) , 
('standarizer', StandardScaler())
 ])
 preprocessor = ColumnTransformer(
 remainder = 'drop',
transformers=[ 
('first', attribs1_pipeline, attribs1), 
('second', attribs2_pipeline, attribs2)
 ])
 full_pipeline = FeatureUnion([ 
('name_pipeline', name_pipeline), 
('title_pipeline', title_pipeline), 
('age_pipeline', age_pipeline), 
('sibsp_pipeline2', sibsp_pipeline2), 
('family_members_pipeline', family_members_pipeline), 
('cabin_pipeline', cabin_pipeline), 
('preprocessor', preprocessor)
 ])
 full_pipeline2 = FeatureUnion([ 
('preprocessor', preprocessor)
 ] )
%matplotlib inline<load_from_csv>
train_set_prepared = full_pipeline.fit_transform(train_set)
 train_set_prepared = full_pipeline2.fit_transform(train_set)
 X_train_prepared = train_set_prepared
y_train_prepared = train_set['Survived']
test_set_prepared = full_pipeline.fit_transform(test_set)
 test_set_prepared = full_pipeline2.fit_transform(test_set)
 X_test_prepared = test_set_prepared
y_test_prepared = test_set['Survived']
test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test = test[test.Date > "2020-04-14"]
all_data = pd.concat([train, test],ignore_index=True ).sort_values(by=['Country_Region','Province_State','Date'])
 all_data['ConfirmedCases'] = all_data['ConfirmedCases'].fillna(-1)
 all_data['Fatalities'] = all_data['Fatalities'].fillna(-0)
 countries = pd.read_csv(".. /input/countries/countries.csv")
 countries.drop(['iso_alpha3','iso_numeric','official_name','name','iso_alpha2'], axis='columns',inplace=True)
 data_quarantine = pd.read_csv(".. /input/countries/quarantine_dates.csv")
 data_quarantine = data_quarantine.groupby("Country" ).max().loc[:,"Start date"]
data_quarantine.loc["Russia"] = "2020-03-30"
countries = countries.set_index("ccse_name", drop=True)
 countries["Quarantine"] = data_quarantine
countries_mean = countries.mean()
all_data = all_data.merge(countries, how ="left" , left_on="Country_Region", right_on='ccse_name')
 all_data['fertility_rate'] = all_data['fertility_rate'].fillna(countries_mean['fertility_rate'])
 all_data['median_age'] = all_data['median_age'].fillna(countries_mean['median_age'])
 all_data['migrants'] = all_data['migrants'].fillna(countries_mean['migrants'])
 all_data['urban_pop_rate'] = all_data['urban_pop_rate'].fillna(countries_mean['urban_pop_rate'])
 all_data['density'] = all_data['density'].fillna(countries_mean['density'])
 all_data['land_area'] = all_data['land_area'].fillna(countries_mean['land_area'])
 all_data['population'] = all_data['population'].fillna(countries_mean['population'])
 all_data['world_share'] = all_data['world_share'].fillna(countries_mean['world_share'])
 all_data['Quarantine'] = all_data['Quarantine'].fillna("2020-04-01")
 all_data.drop(['Id'], axis='columns',inplace=True)
 all_data['Province_State'] = all_data['Province_State'].fillna("zzz" )<data_type_conversions>
rf_model = RandomForestClassifier(random_state=42)
 params_grid = [
{'n_estimators': [100, 200, 300, 400, 500],
'criterion': ['gini', 'entropy'],
'min_samples_split': [2, 3, 4, 5],
'max_features': ['auto', 'log2', None],
'bootstrap': ['True', 'False']}
]
grid_search = GridSearchCV(rf_model, params_grid, cv=5, scoring="accuracy", n_jobs=1)
 grid_search.fit(X_train_prepared, y_train_prepared)
 grid_search.best_params_
data2 = all_data
data2 = data2[data2.ConfirmedCases != 0]
data2.loc[data2.ConfirmedCases == -1,"ConfirmedCases"] = 0
data2["Date"] = pd.to_datetime(data2.Date)
 data4 = data2[["Country_Region","Date"]].groupby("Country_Region" ).min()
data4.columns = ["Date_min"]
data2 = data2.merge(data4, how = 'left', left_on='Country_Region', right_on='Country_Region')
 data2["days"] =(data2.Date - data2.Date_min ).dt.days
data2["days_mart"] =(data2.Date - pd.to_datetime("2020-03-1")).dt.days
data2["days_after_Quarantine"] =(data2.Date - pd.to_datetime(data2.Quarantine)).dt.days
data2.drop(['Date_min'], axis='columns',inplace=True)
 data2.Date = data2["Date"].apply(lambda x: pd.Series(x.strftime("%m-%d")))
 data2 = data2.rename(columns={"ConfirmedCases": "confirmed","Fatalities":"deaths","Country_Region":"countries"})
 data2[data2.countries == "Russia"].iloc[0:70]<filter>
params_grid2 = [
{'n_estimators': [120, 140, 160, 180, 200, 220, 240, 260, 280],
'criterion': ['gini', 'entropy'],
'min_samples_split': [4, 5],
'max_features': ['auto', None],
'bootstrap': ['True']}
]
grid_search2 = GridSearchCV(rf_model, params_grid2, cv=5, scoring="accuracy", n_jobs=1)
 grid_search2.fit(X_train_prepared, y_train_prepared)
 def indices_of_top_k(arr, k):
return np.sort(np.argpartition(np.array(arr), -k)[-k:])
 def fs_calculate_results() :
train_prediction = list()
test_prediction = list()
knn_model = KNeighborsClassifier()
feature_importances = grid_search2.best_estimator_.feature_importances_
for i in range(1,26):
indices_of_top = indices_of_top_k(feature_importances, i)
 X_train_restricted = X_train_prepared[:, indices_of_top]
X_test_restricted = X_test_prepared[:, indices_of_top]
params_grid_fs = [
{'n_neighbors': [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35],
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'metric': ['minkowski', 'euclidean', 'manhattan']}
]
grid_search_fs = GridSearchCV(knn_model, params_grid_fs, cv=5, scoring="accuracy", n_jobs=1)
 grid_search_fs.fit(X_train_restricted, y_train_prepared)
 train_prediction.append(grid_search_fs.best_score_)
 knn_final_model = grid_search_fs.best_estimator_
knn_predictions = knn_final_model.predict(X_test_restricted)
 test_prediction.append(accuracy_score(y_test_prepared, knn_predictions))
return train_prediction, test_prediction
train_pred, test_pred = fs_calculate_results()
fs_results = pd.DataFrame({'train_set': train_pred, 'test_set': test_pred})
 fs_results['train-test_difference'] = fs_results['train_set'] - fs_results['test_set']
fs_results.index += 1
fs_results = fs_results.sort_values('train-test_difference')
 fs_results
data2[data2.countries == "Russia"]<feature_engineering>
feature_importances = grid_search2.best_estimator_.feature_importances_
indices_of_top = indices_of_top_k(feature_importances, 25)
 X_train_prepared = X_train_prepared[:, indices_of_top]
X_test_prepared = X_test_prepared[:, indices_of_top]
knn_model = KNeighborsClassifier()
params_grid = [
{'n_neighbors': [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35],
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'metric': ['minkowski', 'euclidean', 'manhattan']}
]
grid_search = GridSearchCV(knn_model, params_grid, cv=5, scoring="accuracy", n_jobs=1)
 grid_search.fit(X_train_prepared, y_train_prepared)
 grid_search.best_params_
data2.confirmed = np.log10(data2.confirmed+1)
 data2.deaths = np.log10(data2.deaths+1)
 old_con = data2["confirmed"].iloc[:-1]
old_con2 = data2["deaths"].iloc[:-1]
data2 = data2.iloc[1:]
data2["pred_conf"] = old_con.values
data2["pred_deaths"] = old_con2.values
data2 = data2.iloc[1:]
data2["delta_conf1"] = old_con.values[1:] - old_con.values[:-1]
data2["delta_deaths1"] = old_con2.values[1:] - old_con2.values[:-1]
data2 = data2.iloc[1:]
data2["delta_conf2"] = old_con.values[1:-1] - old_con.values[:-2]
data2["delta_deaths2"] = old_con2.values[1:-1] - old_con2.values[:-2]
data2 = data2.iloc[1:]
data2["delta_conf3"] = old_con.values[1:-2] - old_con.values[:-3]
data2["delta_deaths3"] = old_con2.values[1:-2] - old_con2.values[:-3]
data2 = data2.iloc[1:]
data2["delta_conf4"] = old_con.values[1:-3] - old_con.values[:-4]
data2["delta_deaths4"] = old_con2.values[1:-3] - old_con2.values[:-4]
data2.confirmed = data2.confirmed - data2["pred_conf"]
data2.deaths = data2.deaths - data2["pred_deaths"]
data2[data2.countries == "Russia"].iloc[0:70]<drop_column>
print("Training dataset accuracy: ", grid_search.best_score_, sep="")
 knn_model = grid_search.best_estimator_
knn_predictions = knn_model.predict(X_test_prepared)
 print("Test dataset accuracy: ", accuracy_score(y_test_prepared, knn_predictions), sep="" )
days_x = 45
model_Confirmed, model_Death = {}, {}
data3 = data2.drop(['Quarantine','world_share', 'migrants','median_age','land_area','fertility_rate','density','Date'], axis='columns')
 data3.population = data3.population/10**6
one_hot = pd.get_dummies(data3['Province_State'])
 data3 = data3.join(one_hot)
 data_Korea = data3[(data3.countries == 'Korea, South')&(data3.confirmed > 0)]
data_Russia = data3[(data3.countries == 'Russia')&(data3.confirmed > 0)]
new_data = data3[data2.days_mart >= days_x]
old_data = data3[data2.days_mart < days_x]
train_labels = old_data.confirmed
train_death = old_data.deaths
train_data = old_data.drop(['Province_State', 'ForecastId','confirmed','countries','deaths'], axis='columns')
 train_data<predict_on_test>
knn_final_model = KNeighborsClassifier(algorithm='auto', metric='manhattan', n_neighbors=25, weights='uniform')
 knn_final_model.fit(X_train_prepared, y_train_prepared )
def pred_score(models, data, death = False, plot = 0):
predictions_all = 0
plt.rcParams['figure.figsize'] = [20, len(models)*10]
data = data[(data.confirmed > 0)]
death_labels = 10**(data.deaths + data.pred_deaths)- 1
labels = 10**(data.confirmed + data.pred_conf)- 1
data = data.drop(['Province_State','ForecastId','confirmed','countries','deaths'], axis='columns')
 for i, model in enumerate(models):
if model == "reg":
predictions = models[model].predict(scaler.transform(data))
else:
predictions = models[model].predict(data)
 if death:
predictions = predictions + data.pred_deaths.values
else:
predictions = predictions + data.pred_conf.values
predictions = 10**predictions - 1
predictions_all = predictions_all + predictions
if death:
print("Ошибка MALE по смертям", model, np.mean(np.abs(np.log10(( predictions+1)/(death_labels+1)))))
 else:
print("Ошибка MALE по заражениям", model, np.mean(np.abs(np.log10(( predictions+1)/(labels+1)))))
 if plot:
plt.subplot(len(models), 1, i+1)
 if plot==1:
plt.plot(predictions, label = "Предсказанное значение")
 if death:
plt.plot(death_labels.values, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Смерти')
 else:
plt.plot(labels.values, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Заражения')
 if plot==2:
plt.scatter(np.arange(0,len(predictions),1), predictions, s = 1, label = "Предсказанное значение")
 if death:
plt.scatter(np.arange(0,len(predictions),1), death_labels.values, s = 1, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Смерти')
 else:
plt.scatter(np.arange(0,len(predictions),1), labels.values, s = 1, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Заражения')
 plt.title(model)
 plt.grid(True)
 plt.legend()
predictions_all = predictions_all/len(models)
 if death:
print("Ошибка MALE по смертям средняя", np.mean(np.abs(np.log10(( predictions_all+1)/(death_labels+1)))))
 else:
print("Ошибка MALE по заражениям средняя", np.mean(np.abs(np.log10(( predictions_all+1)/(labels+1)))) )<train_on_grid>
titanic_sub_prepared = full_pipeline.fit_transform(titanic_sub)
 titanic_sub_prepared = full_pipeline2.fit_transform(titanic_sub)
 titanic_sub_prepared = titanic_sub_prepared[:, indices_of_top]
titanic_sub_predictions = knn_final_model.predict(titanic_sub_prepared)
 submission = pd.DataFrame({'PassengerId':titanic_sub['PassengerId'], 'Survived':titanic_sub_predictions})
 submission.to_csv('submission.csv', sep=',', index=False )