Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )<load_from_disk>
test_df = pd.read_csv(".. /input/test.csv")
 train_df = pd.read_csv(".. /input/train.csv" )
def print_mse(prd):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(prd, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))<compute_test_metric>
total = train_df.isnull().sum().sort_values(ascending=False)
 percent_1 = train_df.isnull().sum() /train_df.isnull().count() *100
percent_2 =(round(percent_1, 1)).sort_values(ascending=False)
 missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])
 missing_data.head(5 )
print_mse(holdouts_df )<compute_test_metric>
data = [train_df, test_df]
for dataset in data:
dataset['relatives'] = dataset['SibSp'] + dataset['Parch']
dataset.loc[dataset['relatives'] > 0, 'not_alone'] = 0
dataset.loc[dataset['relatives'] == 0, 'not_alone'] = 1
dataset['not_alone'] = dataset['not_alone'].astype(int )
print_mse(holdouts_df[holdouts_df.SN_filter == 1] )<define_variables>
train_df['not_alone'].value_counts()
debug = False<set_options>
train_df = train_df.drop(['PassengerId'], axis=1 )
warnings.filterwarnings('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SEED = 2020
def seed_everything(seed=2020):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 seed_everything(SEED )<compute_test_metric>
deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8}
data = [train_df, test_df]
for dataset in data:
dataset['Cabin'] = dataset['Cabin'].fillna("U0")
 dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 dataset['Deck'] = dataset['Deck'].map(deck)
 dataset['Deck'] = dataset['Deck'].fillna(0)
 dataset['Deck'] = dataset['Deck'].astype(int )
class RMSELoss(nn.Module):
def __init__(self, eps=1e-6):
super().__init__()
self.mse = nn.MSELoss()
self.eps = eps
def forward(self, yhat, y):
loss = torch.sqrt(self.mse(yhat, y)+ self.eps)
 return loss
class MCRMSELoss(nn.Module):
def __init__(self, num_scored=3):
super().__init__()
self.rmse = RMSELoss()
self.num_scored = num_scored
def forward(self, yhat, y):
score = 0
for i in range(self.num_scored):
score += self.rmse(yhat[:, :, i], y[:, :, i])/ self.num_scored
return score<load_from_csv>
train_df = train_df.drop(['Cabin'], axis=1)
 test_df = test_df.drop(['Cabin'], axis=1 )
def load_json(path):
return pd.read_json(path, lines=True)
 df = load_json('/kaggle/input/stanford-covid-vaccine/train.json')
 df_test = load_json('/kaggle/input/stanford-covid-vaccine/test.json')
 sample_sub = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv')
 if debug:
df = df[:200]
df_test = df_test[:200]
df = df[df.SN_filter == 1]<define_variables>
data = [train_df, test_df]
for dataset in data:
mean = train_df["Age"].mean()
std = test_df["Age"].std()
is_null = dataset["Age"].isnull().sum()
rand_age = np.random.randint(mean - std, mean + std, size = is_null)
 age_slice = dataset["Age"].copy()
age_slice[np.isnan(age_slice)] = rand_age
dataset["Age"] = age_slice
dataset["Age"] = train_df["Age"].astype(int )
print(set(df["sequence"].sum()))
print(set(df["structure"].sum()))
print(set(df["predicted_loop_type"].sum()))
sequence_and_structure = [i + j for i in "GACU" for j in "()."]
sequence_and_predicted_loop_type = [i + j for i in "GACU" for j in "XEMBHSI"]
structure_and_predicted_loop_type = [i + j for i in "()." for j in "XEMBHSI"] 
<merge>
train_df["Age"].isnull().sum()
def merge_seq_seq(seq):
half = len(seq)//2
new_seq = []
for i in range(len(seq)//2):
new_seq.append(seq[i] + seq[i+half])
 return new_seq 
<feature_engineering>
common_value = 'S'
data = [train_df, test_df]
for dataset in data:
dataset['Embarked'] = dataset['Embarked'].fillna(common_value )
create_feture = True
preprocess_cols=["sequence", "structure", "predicted_loop_type"]
if create_feture:
df["sequence_and_structure"] =(df["sequence"] + df["structure"] ).apply(merge_seq_seq)
 df_test["sequence_and_structure"] =(df_test["sequence"] + df_test["structure"] ).apply(merge_seq_seq)
 preprocess_cols=["sequence", "structure", "predicted_loop_type", "sequence_and_structure"]
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
df['bpps_sum'] = read_bpps_sum(df)
 df_test['bpps_sum'] = read_bpps_sum(df_test)
 df['bpps_max'] = read_bpps_max(df)
 df_test['bpps_max'] = read_bpps_max(df_test)
 df['bpps_nb'] = read_bpps_nb(df)
 df_test['bpps_nb'] = read_bpps_nb(df_test)
<define_variables>
data = [train_df, test_df]
for dataset in data:
dataset['Fare'] = dataset['Fare'].fillna(0)
 dataset['Fare'] = dataset['Fare'].astype(int )
target_cols = ["reactivity", "deg_Mg_pH10", "deg_Mg_50C"]
tokens = [i for i in "().ACGUBEHIMSX"] + sequence_and_structure + sequence_and_predicted_loop_type + structure_and_predicted_loop_type
token2int = {x:i for i, x in enumerate(tokens)}
def preprocess_inputs(df, cols):
base_fea = np.transpose(
 np.array(
 df[cols].applymap(lambda seq: [token2int[x] for x in seq] ).values.tolist() 
),(0, 2, 1)
)
 if create_feture:
bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 else:
return base_fea
train_inputs = torch.tensor(preprocess_inputs(df, preprocess_cols)).to(device)
 print("input shape: ", train_inputs.shape)
 train_labels = torch.tensor(
 np.array(df[target_cols].values.tolist() ).transpose(0, 2, 1)
 ).float().to(device )<choose_model_class>
data = [train_df, test_df]
titles = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr',\
'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 dataset['Title'] = dataset['Title'].map(titles)
 dataset['Title'] = dataset['Title'].fillna(0 )
models = ["LSTM", "LSTM_short", "GRU", "blend1", "blend2", "blend3", "blend4"]
models = [
{"model": "LSTM", "dropout": 0.4, "embed_dim": 100, "hidden_dim": 128, "hidden_layers": 3},
{"model": "LSTM", "dropout": 0.4, "embed_dim": 100, "hidden_dim": 256, "hidden_layers": 3}
] 
<define_search_model>
train_df = train_df.drop(['Name'], axis=1)
 test_df = test_df.drop(['Name'], axis=1 )
class Wave_Block(nn.Module):
def __init__(self, in_channels, out_channels, dilation_rates, kernel_size):
super(Wave_Block, self ).__init__()
self.num_rates = dilation_rates
self.convs = nn.ModuleList()
self.filter_convs = nn.ModuleList()
self.gate_convs = nn.ModuleList()
self.convs.append(nn.Conv1d(in_channels, out_channels, kernel_size=1))
dilation_rates = [2 ** i for i in range(dilation_rates)]
for dilation_rate in dilation_rates:
self.filter_convs.append(
 nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=int(( dilation_rate*(kernel_size-1)) /2), dilation=dilation_rate))
self.gate_convs.append(
 nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=int(( dilation_rate*(kernel_size-1)) /2), dilation=dilation_rate))
self.convs.append(nn.Conv1d(out_channels, out_channels, kernel_size=1))
def forward(self, x):
x = self.convs[0](x)
 res = x
for i in range(self.num_rates):
x = torch.tanh(self.filter_convs[i](x)) * torch.sigmoid(self.gate_convs[i](x))
x = self.convs[i + 1](x)
 res = res + torch.nn.functional.pad(x,(0, 1, 0, 0))
return res
class MyModel(nn.Module):
def __init__(
 self, seq_len=107, pred_len=68, dropout=0.4, embed_dim=100, hidden_dim=64, hidden_layers=3, main_layer="LSTM" 
):
super(MyModel, self ).__init__()
self.pred_len = pred_len
self.main_layer = main_layer
if main_layer == "GRU":
hidden_dim //= 3
self.embeding = nn.Embedding(num_embeddings=len(token2int), embedding_dim=embed_dim)
 self.lstm_layer = nn.LSTM(
 input_size=embed_dim * len(preprocess_cols)+ 3,
hidden_size=hidden_dim,
num_layers=hidden_layers,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.gru_layer = nn.GRU(
 input_size=embed_dim * len(preprocess_cols)+ 3,
hidden_size=hidden_dim,
num_layers=hidden_layers,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.hidden_lstm_layer = nn.LSTM(
 input_size=hidden_dim*2,
hidden_size=hidden_dim,
num_layers=1,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.hidden_gru_layer = nn.GRU(
 input_size=hidden_dim*2,
hidden_size=hidden_dim,
num_layers=1,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.wave_block = Wave_Block(hidden_dim*2, hidden_dim*2, 12, hidden_dim)
 self.linear1 = nn.Linear(hidden_dim * 2, hidden_dim)
 self.linear2 = nn.Linear(hidden_dim, hidden_dim)
 self.linear3 = nn.Linear(hidden_dim, len(target_cols))
def forward(self, seqs):
if create_feture:
categorical_feats = seqs[:, :, :len(preprocess_cols)].long()
numerical_feats = seqs[:, :, len(preprocess_cols):].float()
embed = self.embeding(categorical_feats)
 reshaped = torch.reshape(embed,(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = torch.cat([reshaped, numerical_feats], axis=2)
 else:
embed = self.embeding(seqs)
 reshaped = torch.reshape(embed,(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = torch.cat([reshaped, numerical_feats], axis=2)
 if self.main_layer == "LSTM":
output, hidden = self.lstm_layer(reshaped)
 elif self.main_layer == "GRU":
output, hidden = self.gru_layer(reshaped)
 elif self.main_layer == "blend1":
output, hidden = self.lstm_layer(reshaped)
 output, hidden = self.hidden_lstm_layer(output)
 elif self.main_layer == "blend2":
output, hidden = self.gru_layer(reshaped)
 output, hidden = self.hidden_gru_layer(output)
 elif self.main_layer == "blend3":
output, hidden = self.gru_layer(reshaped)
 output, hidden = self.hidden_gru_layer(output)
 elif self.main_layer == "blend4":
output, hidden = self.lstm_layer(reshaped)
 output, hidden = self.hidden_lstm_layer(output)
 elif self.main_layer == "WaveNet":
output, hidden = self.lstm_layer(reshaped)
 output = output.permute(0, 2, 1)
 output = self.wave_block(output)
 output = output.permute(0, 2, 1)
 truncated = output[:, : self.pred_len, :]
truncated = self.linear1(truncated)
 truncated = self.linear2(truncated)
 truncated = self.linear2(truncated)
 out = self.linear3(truncated)
 return out
criterion = MCRMSELoss(len(target_cols))
def compute_loss(batch_X, batch_Y, model, optimizer=None, is_train=True, scheduler=None):
model.train(is_train)
 pred_Y = model(batch_X)
 loss = criterion(pred_Y, batch_Y)
 if is_train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
if scheduler:
scheduler.step()
return loss.item() , pred_Y 
<define_variables>
genders = {"male": 0, "female": 1}
data = [train_df, test_df]
for dataset in data:
dataset['Sex'] = dataset['Sex'].map(genders )
FOLDS = 4
EPOCHS = 100
if debug:
EPOCHS = 2
BATCH_SIZE = 64
VERBOSE = 2
LR = 0.016<create_dataframe>
train_df = train_df.drop(['Ticket'], axis=1)
 test_df = test_df.drop(['Ticket'], axis=1 )
public_df = df_test.query("seq_length == 107" ).copy()
private_df = df_test.query("seq_length == 130" ).copy()
public_inputs = torch.tensor(preprocess_inputs(public_df, preprocess_cols)).to(device)
 private_inputs = torch.tensor(preprocess_inputs(private_df, preprocess_cols)).to(device)
 public_loader = DataLoader(TensorDataset(public_inputs), shuffle=False, batch_size=BATCH_SIZE)
 private_loader = DataLoader(TensorDataset(private_inputs), shuffle=False, batch_size=BATCH_SIZE)
<train_model>
ports = {"S": 0, "C": 1, "Q": 2}
data = [train_df, test_df]
for dataset in data:
dataset['Embarked'] = dataset['Embarked'].map(ports )
if debug:
kmeans_model = KMeans(n_clusters=50, random_state=110 ).fit(preprocess_inputs(df, preprocess_cols)[:,:,0])
 else:
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(df, preprocess_cols)[:,:,0])
 kmeans_labels = kmeans_model.labels_<init_hyperparams>
data = [train_df, test_df]
for dataset in data:
dataset['Age'] = dataset['Age'].astype(int)
 dataset.loc[ dataset['Age'] <= 11, 'Age'] = 0
dataset.loc[(dataset['Age'] > 11)&(dataset['Age'] <= 18), 'Age'] = 1
dataset.loc[(dataset['Age'] > 18)&(dataset['Age'] <= 22), 'Age'] = 2
dataset.loc[(dataset['Age'] > 22)&(dataset['Age'] <= 27), 'Age'] = 3
dataset.loc[(dataset['Age'] > 27)&(dataset['Age'] <= 33), 'Age'] = 4
dataset.loc[(dataset['Age'] > 33)&(dataset['Age'] <= 40), 'Age'] = 5
dataset.loc[(dataset['Age'] > 40)&(dataset['Age'] <= 66), 'Age'] = 6
dataset.loc[ dataset['Age'] > 66, 'Age'] = 6
model_histories = {str(model_id): [] for model_id in models}
model_oof_preds = {str(model_id): np.zeros(( df.shape[0], 68, len(target_cols)))for model_id in models}
model_private_preds = {str(model_id): np.zeros(( private_df.shape[0], 130, len(target_cols)))for model_id in models}
model_public_preds = {str(model_id): np.zeros(( public_df.shape[0], 107, len(target_cols)))for model_id in models}
criterion = MCRMSELoss()
gkf = GroupKFold(FOLDS)
 for model_dict in models:
model_id = str(model_dict)
 model_name = model_dict["model"]
dropout = model_dict["dropout"]
embed_dim = model_dict["embed_dim"]
hidden_layers = model_dict["hidden_layers"]
model_oof_pred = []
oof_idxes = []
for k,(train_index, val_index)in enumerate(gkf.split(train_inputs, df["reactivity"], kmeans_labels)) :
oof_idxes.append(val_index)
 train_dataset = TensorDataset(train_inputs[train_index], train_labels[train_index])
 val_dataset = TensorDataset(train_inputs[val_index], train_labels[val_index])
 train_loader = DataLoader(train_dataset, shuffle=True, batch_size=BATCH_SIZE)
 val_loader = DataLoader(val_dataset, shuffle=True, batch_size=BATCH_SIZE)
 model = MyModel(dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 optimizer = optim.Adam(model.parameters() , lr=LR)
 scheduler = None
scheduler = CosineAnnealingLR(optimizer, T_max=20, eta_min=0.008)
 train_losses = []
val_losses = []
for epoch in tqdm(range(EPOCHS)) :
train_losses_batch = []
val_losses_batch = []
for(batch_X, batch_Y)in train_loader:
train_loss, _ = compute_loss(batch_X, batch_Y, model, optimizer=optimizer, is_train=True, scheduler=scheduler)
 train_losses_batch.append(train_loss)
 for(batch_X, batch_Y)in val_loader:
model.eval()
val_loss, val_pred = compute_loss(batch_X, batch_Y, model, optimizer=optimizer, is_train=False)
 val_losses_batch.append(val_loss)
 train_losses.append(np.mean(train_losses_batch))
val_losses.append(np.mean(val_losses_batch))
model_state = model.state_dict()
torch.save(model_state, f"{model_id}_fold{k}.pth")
 del model
model_histories[model_id].append({"train_loss": train_losses, "val_loss": val_losses})
 model_short = MyModel(seq_len=107, pred_len=107, dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 model_short.load_state_dict(model_state)
 model_short.eval()
model_public_pred = np.ndarray(( 0, 107, len(target_cols)))
 for batch in public_loader:
batch_X = batch[0]
pred = model_short(batch_X ).detach().cpu().numpy()
model_public_pred = np.concatenate([model_public_pred, pred], axis=0)
 model_public_preds[model_id] += model_public_pred / FOLDS
model_long = MyModel(seq_len=130, pred_len=130, dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 model_long.load_state_dict(model_state)
 model_long.eval()
model_private_pred = np.ndarray(( 0, 130, len(target_cols)))
 for batch in private_loader:
batch_X = batch[0]
pred = model_long(batch_X ).detach().cpu().numpy()
model_private_pred = np.concatenate([model_private_pred, pred], axis=0)
 model_private_preds[model_id] += model_private_pred / FOLDS
oof_loader = DataLoader(val_dataset, shuffle=False, batch_size=BATCH_SIZE)
 model_oof = MyModel(seq_len=107, pred_len=68, dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 model_oof.load_state_dict(model_state)
 model_oof.eval()
oof_pred = np.ndarray(( 0, 68, len(target_cols)))
 for batch in oof_loader:
batch_X = batch[0]
pred = model_oof(batch_X ).detach().cpu().numpy()
oof_pred = np.concatenate([oof_pred, pred], axis=0)
 model_oof_pred.append(oof_pred)
 del model_short, model_long
gc.collect()
oof_idxes = np.concatenate(oof_idxes)
 order = np.argsort(oof_idxes)
 model_oof_preds[model_id] = np.concatenate(model_oof_pred)[order] 
<define_variables>
train_df['Age'].value_counts()
public_df = df_test.query("seq_length == 107" ).copy()
private_df = df_test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df, preprocess_cols)
 private_inputs = preprocess_inputs(private_df, preprocess_cols )<prepare_output>
data = [train_df, test_df]
for dataset in data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[(dataset['Fare'] > 31)&(dataset['Fare'] <= 99), 'Fare'] = 3
dataset.loc[(dataset['Fare'] > 99)&(dataset['Fare'] <= 250), 'Fare'] = 4
dataset.loc[ dataset['Fare'] > 250, 'Fare'] = 5
dataset['Fare'] = dataset['Fare'].astype(int )
preds_model = {str(model_id): [] for model_id in models}
submissions = {}
for model_dict in models:
model_id = str(model_dict)
 for df, preds in [(public_df, model_public_preds[model_id]),(private_df, model_private_preds[model_id])]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_model[model_id].append(single_df)
 preds_df = pd.concat(preds_model[model_id])
 print(preds_df.head())
 submission = sample_sub[['id_seqpos']].merge(preds_df, on=['id_seqpos'])
 submission['deg_pH10'] = 0
submission['deg_50C'] = 0
print(submission.head())
 submission.to_csv(f'submission_{model_id}.csv', index=False)
 submissions[model_id] = submission
print('Submission saved')
<save_to_csv>
data = [train_df, test_df]
for dataset in data:
dataset['Age_Class']= dataset['Age']* dataset['Pclass']
if True:
id_seqpos = submissions[str(models[0])]["id_seqpos"]
ensemble_submission = submissions[str(models[0])].drop("id_seqpos",axis=1)*best_alpha + submissions[str(models[1])].drop("id_seqpos",axis=1)*(1-best_alpha)
 ensemble_submission["id_seqpos"] = id_seqpos
ensemble_submission.to_csv(f'submission_lstm_gru_ensemble.csv', index=False )<train_model>
for dataset in data:
dataset['Fare_Per_Person'] = dataset['Fare']/(dataset['relatives']+1)
 dataset['Fare_Per_Person'] = dataset['Fare_Per_Person'].astype(int )
if not debug:
!curl -X POST -H 'Content-type: application/json' --data '{"text":"commit done! "}' <your_webhook_url><import_modules>
train_df.head(20 )
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedShuffleSplit
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten, Convolution1D, Dropout
from keras.optimizers import SGD
from keras.utils import np_utils 
<load_from_csv>
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
train = pd.read_csv('.. /input/leaf-classification/train.csv.zip')
 test = pd.read_csv('.. /input/leaf-classification/test.csv.zip')
<categorify>
sgd = linear_model.SGDClassifier(max_iter=5, tol=None)
 sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 sgd.score(X_train, Y_train)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 print(round(acc_sgd,2,), "%" )
def encode(train, test):
label_encoder = LabelEncoder().fit(train.species)
 labels = label_encoder.transform(train.species)
 classes = list(label_encoder.classes_)
 train = train.drop(['species', 'id'], axis=1)
 test_ids=test.id
test = test.drop('id', axis=1)
 return train, labels, test, classes,test_ids<categorify>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_prediction = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 print(round(acc_random_forest,2,), "%" )
train, labels, test, classes,test_ids = encode(train, test)
<normalization>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 print(round(acc_log,2,), "%" )
scaler = StandardScaler().fit(train.values)
 scaled_train = scaler.transform(train.values )<split>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 print(round(acc_knn,2,), "%" )
sss = StratifiedShuffleSplit(test_size=0.1, random_state=23)
 for train_index, valid_index in sss.split(scaled_train, labels):
X_train, X_valid = scaled_train[train_index], scaled_train[valid_index]
y_train, y_valid = labels[train_index], labels[valid_index] 
<define_variables>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 print(round(acc_gaussian,2,), "%" )
nb_features = 64
nb_class = len(classes )<prepare_x_and_y>
perceptron = Perceptron(max_iter=5)
 perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 print(round(acc_perceptron,2,), "%" )
X_train_r = np.zeros(( len(X_train), nb_features, 3))
X_train_r[:, :, 0] = X_train[:, :nb_features]
X_train_r[:, :, 1] = X_train[:, nb_features:128]
X_train_r[:, :, 2] = X_train[:, 128:]
X_valid_r = np.zeros(( len(X_valid), nb_features, 3))
X_valid_r[:, :, 0] = X_valid[:, :nb_features]
X_valid_r[:, :, 1] = X_valid[:, nb_features:128]
X_valid_r[:, :, 2] = X_valid[:, 128:] 
<choose_model_class>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 print(round(acc_linear_svc,2,), "%" )
model = Sequential()
model.add(Convolution1D(512, 1, input_shape=(nb_features, 3)))
 model.add(Activation('relu'))
model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(2048, activation='relu'))
model.add(Dense(1024, activation='relu'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))<train_model>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 print(round(acc_decision_tree,2,), "%" )
y_train = np_utils.to_categorical(y_train, nb_class)
 y_valid = np_utils.to_categorical(y_valid, nb_class)
 sgd = SGD(lr=0.01, nesterov=True, decay=1e-6, momentum=0.9)
 model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])
 nb_epoch = 15
model.fit(X_train_r, y_train, epochs=nb_epoch, validation_data=(X_valid_r, y_valid), batch_size=16 )<normalization>
rf = RandomForestClassifier(n_estimators=100)
 scores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = "accuracy" )
scaler = StandardScaler().fit(test.values)
 scaled_test = scaler.transform(test.values )<prepare_x_and_y>
print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std() )
test_dataset = np.zeros(( len(scaled_test), nb_features, 3))
test_dataset[:, :, 0] = scaled_test[:, :nb_features]
test_dataset[:, :, 1] = scaled_test[:, nb_features:128]
test_dataset[:, :, 2] = scaled_test[:, 128:]<predict_on_test>
importances = pd.DataFrame({'feature':X_train.columns,'importance':np.round(random_forest.feature_importances_,3)})
 importances = importances.sort_values('importance',ascending=False ).set_index('feature' )
preds_test = model.predict_proba(test_dataset)
 preds_test<prepare_output>
train_df = train_df.drop("not_alone", axis=1)
 test_df = test_df.drop("not_alone", axis=1)
 train_df = train_df.drop("Parch", axis=1)
 test_df = test_df.drop("Parch", axis=1 )
submission = pd.DataFrame(preds_test, columns=classes)
 submission.insert(0, 'id', test_ids)
 submission<save_to_csv>
random_forest = RandomForestClassifier(n_estimators=100, oob_score = True)
 random_forest.fit(X_train, Y_train)
 Y_prediction = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 print(round(acc_random_forest,2,), "%" )
submission.to_csv('submission.csv', index=False)
 print('done!' )<import_modules>
print("oob score:", round(random_forest.oob_score_, 4)*100, "%" )
import pandas as pd
import numpy as np
from sklearn.model_selection import GridSearchCV
import xgboost as xgb
from sklearn.model_selection import cross_val_score
from sklearn.metrics import log_loss
from sklearn.preprocessing import LabelBinarizer
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler<load_from_csv>
random_forest = RandomForestClassifier(criterion = "gini",
min_samples_leaf = 1,
min_samples_split = 10,
n_estimators=100,
max_features='auto',
oob_score=True,
random_state=1,
n_jobs=-1)
 random_forest.fit(X_train, Y_train)
 Y_prediction = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 print("oob score:", round(random_forest.oob_score_, 4)*100, "%" )
train = pd.read_csv('.. /input/leaf-classification/train.csv.zip' , index_col = False)
 train<drop_column>
predictions = cross_val_predict(random_forest, X_train, Y_train, cv=3)
 confusion_matrix(Y_train, predictions )
x_train = train.drop(['id', 'species'], axis=1 ).values<categorify>
print("Precision:", precision_score(Y_train, predictions))
print("Recall:",recall_score(Y_train, predictions))
le = LabelEncoder().fit(train['species'])
 y_train = le.transform(train['species'] )<normalization>
f1_score(Y_train, predictions )
scaler = StandardScaler().fit(x_train)
 x_train = scaler.transform(x_train )<train_model>
y_scores = random_forest.predict_proba(X_train)
 y_scores = y_scores[:,1]
precision, recall, threshold = precision_recall_curve(Y_train, y_scores )
clf = LogisticRegression(solver='lbfgs', multi_class='multinomial')
 clf.fit(x_train, y_train )<load_from_csv>
false_positive_rate, true_positive_rate, thresholds = roc_curve(Y_train, y_scores )
test = pd.read_csv('.. /input/leaf-classification/test.csv.zip' ,index_col = False)
 test<prepare_x_and_y>
r_a_score = roc_auc_score(Y_train, y_scores)
 print("ROC-AUC-Score:", r_a_score )
test_ids = test.pop('id')
 x_test = test.values<prepare_x_and_y>
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_prediction
})
 submission.to_csv('submission.csv', index=False )