Kaggle competition: Titanic - Machine Learning from Disaster
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<concatenate>
df = pd.read_csv("/kaggle/input/titanic/train.csv")
 df
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )<normalization>
test = pd.read_csv(".. /input/titanic/test.csv")
 test
def petrosian_fd(x):

 n = len(x)
 diff = np.ediff1d(x)
 N_delta =(diff[1:-1] * diff[0:-2] < 0 ).sum()
return np.log10(n)/(np.log10(n)+ np.log10(n /(n + 0.4 * N_delta)))
 def katz_fd(x):

 x = np.array(x)
 dists = np.abs(np.ediff1d(x))
ll = dists.sum()
ln = np.log10(np.divide(ll, dists.mean()))
aux_d = x - x[0]
d = np.max(np.abs(aux_d[1:]))
return np.divide(ln, np.add(ln, np.log10(np.divide(d, ll))))
@jit('UniTuple(float64, 2 )(float64[:], float64[:])', nopython=True)
 def _linear_regression(x, y):

 n_times = x.size
sx2 = 0
sx = 0
sy = 0
sxy = 0
for j in range(n_times):
sx2 += x[j] ** 2
sx += x[j]
sxy += x[j] * y[j]
sy += y[j]
den = n_times * sx2 -(sx ** 2)
 num = n_times * sxy - sx * sy
slope = num / den
intercept = np.mean(y)- slope * np.mean(x)
 return slope, intercept
@jit('i8[:](f8, f8, f8)', nopython=True)
 def _log_n(min_n, max_n, factor):

 max_i = int(floor(log(1.0 * max_n / min_n)/ log(factor)))
 ns = [min_n]
for i in range(max_i + 1):
n = int(floor(min_n *(factor ** i)))
 if n > ns[-1]:
ns.append(n)
 return np.array(ns, dtype=np.int64)
 @jit('float64(float64[:], int32)')
 def _higuchi_fd(x, kmax):

 n_times = x.size
lk = np.empty(kmax)
 x_reg = np.empty(kmax)
 y_reg = np.empty(kmax)
 for k in range(1, kmax + 1):
lm = np.empty(( k,))
for m in range(k):
ll = 0
n_max = floor(( n_times - m - 1)/ k)
 n_max = int(n_max)
 for j in range(1, n_max):
ll += abs(x[m + j * k] - x[m +(j - 1)* k])
 ll /= k
ll *=(n_times - 1)/(k * n_max)
 lm[m] = ll
m_lm = 0
for m in range(k):
m_lm += lm[m]
m_lm /= k
lk[k - 1] = m_lm
x_reg[k - 1] = log(1./ k)
 y_reg[k - 1] = log(m_lm)
 higuchi, _ = _linear_regression(x_reg, y_reg)
 return higuchi
def higuchi_fd(x, kmax=10):

 x = np.asarray(x, dtype=np.float64)
 kmax = int(kmax)
 return _higuchi_fd(x, kmax )<categorify>
for col in df.columns:
print(str(col)+ ":" + str(len(df[col].unique())) )
def _embed(x, order=3, delay=1):

 N = len(x)
 if order * delay > N:
raise ValueError("Error: order * delay should be lower than x.size")
 if delay < 1:
raise ValueError("Delay has to be at least 1.")
 if order < 2:
raise ValueError("Order has to be at least 2.")
 Y = np.zeros(( order, N -(order - 1)* delay))
for i in range(order):
Y[i] = x[i * delay:i * delay + Y.shape[1]]
return Y.T
all = ['perm_entropy', 'spectral_entropy', 'svd_entropy', 'app_entropy',
'sample_entropy']
def perm_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 ran_order = range(order)
 hashmult = np.power(order, ran_order)
 sorted_idx = _embed(x, order=order, delay=delay ).argsort(kind='quicksort')
 hashval =(np.multiply(sorted_idx, hashmult)).sum(1)
 _, c = np.unique(hashval, return_counts=True)
 p = np.true_divide(c, c.sum())
 pe = -np.multiply(p, np.log2(p)).sum()
if normalize:
pe /= np.log2(factorial(order))
return pe
def spectral_entropy(x, sf, method='fft', nperseg=None, normalize=False):

 x = np.array(x)
 if method == 'fft':
_, psd = periodogram(x, sf)
 elif method == 'welch':
_, psd = welch(x, sf, nperseg=nperseg)
 psd_norm = np.divide(psd, psd.sum())
 se = -np.multiply(psd_norm, np.log2(psd_norm)).sum()
if normalize:
se /= np.log2(psd_norm.size)
 return se
def svd_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 mat = _embed(x, order=order, delay=delay)
 W = np.linalg.svd(mat, compute_uv=False)
 W /= sum(W)
 svd_e = -np.multiply(W, np.log2(W)).sum()
if normalize:
svd_e /= np.log2(order)
 return svd_e
def _app_samp_entropy(x, order, metric='chebyshev', approximate=True):

 _all_metrics = KDTree.valid_metrics
if metric not in _all_metrics:
raise ValueError('The given metric(%s)is not valid.The valid '
'metric names are: %s' %(metric, _all_metrics))
phi = np.zeros(2)
 r = 0.2 * np.std(x, axis=-1, ddof=1)
 _emb_data1 = _embed(x, order, 1)
 if approximate:
emb_data1 = _emb_data1
else:
emb_data1 = _emb_data1[:-1]
count1 = KDTree(emb_data1, metric=metric ).query_radius(emb_data1, r,
count_only=True
).astype(np.float64)
 emb_data2 = _embed(x, order + 1, 1)
 count2 = KDTree(emb_data2, metric=metric ).query_radius(emb_data2, r,
count_only=True
).astype(np.float64)
 if approximate:
phi[0] = np.mean(np.log(count1 / emb_data1.shape[0]))
phi[1] = np.mean(np.log(count2 / emb_data2.shape[0]))
else:
phi[0] = np.mean(( count1 - 1)/(emb_data1.shape[0] - 1))
phi[1] = np.mean(( count2 - 1)/(emb_data2.shape[0] - 1))
return phi
@jit('f8(f8[:], i4, f8)', nopython=True)
 def _numba_sampen(x, mm=2, r=0.2):

 n = x.size
n1 = n - 1
mm += 1
mm_dbld = 2 * mm
r *= x.std()
run = [0] * n
run1 = run[:]
r1 = [0] *(n * mm_dbld)
 a = [0] * mm
b = a[:]
p = a[:]
for i in range(n1):
nj = n1 - i
for jj in range(nj):
j = jj + i + 1
if abs(x[j] - x[i])< r:
run[jj] = run1[jj] + 1
m1 = mm if mm < run[jj] else run[jj]
for m in range(m1):
a[m] += 1
if j < n1:
b[m] += 1
else:
run[jj] = 0
for j in range(mm_dbld):
run1[j] = run[j]
r1[i + n * j] = run[j]
if nj > mm_dbld - 1:
for j in range(mm_dbld, nj):
run1[j] = run[j]
m = mm - 1
while m > 0:
b[m] = b[m - 1]
m -= 1
b[0] = n * n1 / 2
a = np.array([float(aa)for aa in a])
 b = np.array([float(bb)for bb in b])
 p = np.true_divide(a, b)
 return -log(p[-1])
 def app_entropy(x, order=2, metric='chebyshev'):

 phi = _app_samp_entropy(x, order=order, metric=metric, approximate=True)
 return np.subtract(phi[0], phi[1])
 def sample_entropy(x, order=2, metric='chebyshev'):

 x = np.asarray(x, dtype=np.float64)
 if metric == 'chebyshev' and x.size < 5000:
return _numba_sampen(x, mm=order, r=0.2)
 else:
phi = _app_samp_entropy(x, order=order, metric=metric,
approximate=False)
 return -np.log(np.divide(phi[1], phi[0]))<compute_test_metric>
df["Age"].dropna()
def entropy_and_fractal_dim(x):
return [perm_entropy(x), svd_entropy(x), app_entropy(x), sample_entropy(x), petrosian_fd(x), katz_fd(x), higuchi_fd(x)]<categorify>
dmean = df["Age"].dropna().mean()
dmean
signals = X.reshape(( len(X), X.shape[1]*X.shape[2]))
features = []
for signal in signals:
features.append(entropy_and_fractal_dim(signal))<normalization>
df["Age"] = df["Age"].fillna(dmean )
features = np.array(features ).reshape(( len(features), 7))
scaler = MinMaxScaler(feature_range=(0, 1))
scaler.fit(features)
 features = scaler.transform(features )<choose_model_class>
test["Age"] = test["Age"].fillna(dmean )
def model_lstm(input_shape, feat_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
feat = Input(shape=(feat_shape[1],))
bi_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True), merge_mode='concat' )(inp)
 bi_gru = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat' )(bi_lstm)
 attention = Attention(input_shape[1] )(bi_gru)
 x = concatenate([attention, feat], axis=1)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp, feat], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model<split>
from sklearn.preprocessing import LabelEncoder
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=10 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_feat, train_y, val_X, val_feat, val_y = X[train_idx], features[train_idx], y[train_idx], X[val_idx], features[val_idx], y[val_idx]
model = model_lstm(train_X.shape, features.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit([train_X, train_feat], train_y, batch_size=128, epochs=50, validation_data=([val_X, val_feat], val_y), callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict([val_X, val_feat], batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape 
<find_best_params>
le=LabelEncoder()
le.fit(df["Sex"])
 df["Sex"] = le.transform(df["Sex"])
 test["Sex"] = le.transform(test["Sex"] )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(
 K.variable(y_true.astype("float64")) , K.variable(( y_proba > threshold ).astype("float64"))))
if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result<compute_test_metric>
dfall = pd.concat([df,test],axis=0)
 dfall
optimal_values = threshold_search(y_val, preds_val)
 best_threshold = optimal_values['threshold']
best_score = optimal_values['matthews_correlation']<compute_test_metric>
dfall2 = pd.get_dummies(dfall["Embarked"],dummy_na=True)
 dfall2
print("Optimal Threshold : " + str(best_threshold))
print("Best Matthews Correlation : " + str(best_score))<load_from_csv>
dfall = pd.concat([dfall,dfall2],axis=1)
 dfall
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )<drop_column>
train = dfall.iloc[:len(df),:]
test = dfall.iloc[len(df):,:]
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<define_variables>
from sklearn import preprocessing
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )<concatenate>
folds = train.copy()
Fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds["Survived"])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.groupby(['fold', "Survived"] ).size() )
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape<normalization>
p_train = folds[folds["fold"] != 0]
p_val = folds[folds["fold"] == 0]
signals = X_test_input.reshape(( len(X_test_input), X_test_input.shape[1]*X_test_input.shape[2]))
features_test = []
for signal in signals:
features_test.append(entropy_and_fractal_dim(signal))<normalization>
p_train = p_train.reset_index(drop=True)
 p_val = p_val.reset_index(drop=True )
features_test = np.array(features_test ).reshape(( len(features_test), 7))
features_test = scaler.transform(features_test)
 np.save("features_test.npy",features_test )<load_from_csv>
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
FEATURES = ["Pclass","Sex","Age","SibSp","Parch","C","Q","S",np.nan]
TARGET = "Survived"
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict([X_test_input, features_test], batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )<data_type_conversions>
train_X = np.array(p_train[FEATURES])
 train_Y = np.array(p_train[TARGET])
 val_X = np.array(p_val[FEATURES])
 val_Y = np.array(p_val[TARGET] )
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
scaler = StandardScaler()
scaler.fit(train_X)
 train_X = scaler.transform(train_X)
 val_X = scaler.transform(val_X )
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
train_X = torch.from_numpy(train_X ).float()
train_Y = torch.from_numpy(train_Y ).long()
val_X = torch.from_numpy(val_X ).float()
val_Y = torch.from_numpy(val_Y ).long()
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from joblib import Parallel, delayed
import tensorflow as tf
from tensorflow import set_random_seed
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from sklearn.metrics import matthews_corrcoef
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *<define_variables>
train_dataset = TensorDataset(train_X,train_Y)
 val_dataset = TensorDataset(val_X,val_Y )
N_SPLITS = 5
sample_size = 800000
MAX_THREADS = 2
RANDOM_SEED = 2019<define_variables>
train_dataloader = DataLoader(train_dataset,batch_size=128,shuffle = True)
 val_dataloader = DataLoader(val_dataset,batch_size=32,shuffle = False )
np.random.seed(RANDOM_SEED)
 set_random_seed(RANDOM_SEED )<normalization>
for a in train_dataloader:
print(a)
 break
class StatefullMCC(Layer):
def __init__(self, thresholds, **kwargs):
super(StatefullMCC, self ).__init__(**kwargs)
 self.thresholds = thresholds
self.stateful = True
self.name='matthews_correlation'
def reset_states(self):
K.get_session().run(tf.variables_initializer(self.local_variable))
def metric_variable(self, shape, dtype, validate_shape=True, name=None):
return tf.Variable(
 np.zeros(shape),
dtype=dtype,
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES],
validate_shape=validate_shape,
name=name, 
)
 def initialize_vars(self, y_true, y_pred):
self.tp = [None]*len(self.thresholds)
 self.tn = [None]*len(self.thresholds)
 self.fp = [None]*len(self.thresholds)
 self.fn = [None]*len(self.thresholds)
 for i,_ in enumerate(self.thresholds):
self.tp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tp%d'%i)
 self.tn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tn%d'%i)
 self.fp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fp%d'%i)
 self.fn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fn%d'%i)
 tp_op = tf.assign_add(self.tp[i], tf.count_nonzero(y_true * y_pred[i], axis=0))
self.add_update(tp_op)
 tn_op = tf.assign_add(self.tn[i], tf.count_nonzero(( 1-y_true)*(1-y_pred[i]), axis=0))
self.add_update(tn_op)
 fp_op = tf.assign_add(self.fp[i], tf.count_nonzero(( 1-y_true)*y_pred[i], axis=0))
self.add_update(fp_op)
 fn_op = tf.assign_add(self.fn[i], tf.count_nonzero(y_true*(1-y_pred[i]), axis=0))
self.add_update(fn_op)
 self.local_variable = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES)
 def __call__(self, y_true, y_pred):
rounded_preds = [K.cast(K.greater_equal(y_pred, t), 'float32')for t in self.thresholds]
self.initialize_vars(y_true, rounded_preds)
 mcc_vec = []
for i,_ in enumerate(self.thresholds):
num = tf.cast(( self.tp[i] * self.tn[i] - self.fp[i] * self.fn[i]), 'float32')
 den = K.sqrt(tf.cast(( self.tp[i] + self.fp[i])*(self.tp[i] + self.fn[i])*(self.tn[i] + self.fp[i])*(self.tn[i] + self.fn[i]), 'float32'))
mcc_vec.append(num/(den + tf.constant(K.epsilon())))
 return K.max(mcc_vec )<define_search_model>
class Net(nn.Module):
def __init__(self):
super(Net,self ).__init__()
self.fc1 = nn.Linear(len(FEATURES),512)
 self.fc2 = nn.Linear(512,256)
 self.fc3 = nn.Linear(256,2)
 def forward(self,x):
x= F.relu(self.fc1(x))
x= F.relu(self.fc2(x))
x = self.fc3(x)
 return x
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.glorot_uniform(RANDOM_SEED)
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
model=Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , lr=0.01 )
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<load_from_csv>
total_loss = 0
model.train()
for train_x,train_y in train_dataloader:
print(train_x)
 print(train_y)
 train_x, train_y = Variable(train_x),Variable(train_y)
 optimizer.zero_grad()
output = model(train_x)
 loss = criterion(output,train_y)
 loss.backward()
optimizer.step()
total_loss += loss.item()
break
def get_features(dataset='train', split_parts=10):
if dataset == 'train':
cache_file = 'X.npy'
meta_file = '.. /input/metadata_train.csv'
elif dataset == 'test':
cache_file = 'X_test.npy'
meta_file = '.. /input/metadata_test.csv'
if os.path.isfile(cache_file):
X = np.load(cache_file)
 y = None
if dataset == 'train':
y = np.load('y.npy')
 else:
meta_df = pd.read_csv(meta_file)
 data_measurements = meta_df.pivot(index='id_measurement', columns='phase', values='signal_id')
 data_measurements = data_measurements.values
data_measurements = np.array_split(data_measurements, split_parts, axis=0)
 X = Parallel(n_jobs=min(split_parts, MAX_THREADS), verbose=1 )(delayed(prep_data )(p, dataset)for p in data_measurements)
 try:
y = meta_df.loc[meta_df['phase']==0, 'target'].values
except:
y = None
X = np.concatenate(X, axis=0)
 if dataset == 'train':
np.save("X.npy",X)
 np.save("y.npy",y)
 elif dataset == 'test':
np.save("X_test.npy",X)
 return X, y 
<define_variables>
def training(train_dataloader,model):
total_loss = 0
model.train()
for train_x,train_y in train_dataloader:
train_x, train_y = Variable(train_x),Variable(train_y)
 optimizer.zero_grad()
output = model(train_x)
 loss = criterion(output,train_y)
 loss.backward()
optimizer.step()
total_loss += loss.item()
return model,total_loss
max_num = 127
min_num = -128<categorify>
total_valloss = 0
model.eval()
with torch.no_grad() :
for val_x,val_y in val_dataloader:
output = model(val_x)
 valloss = criterion(output,val_y)
 total_valloss += valloss.item()
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
def valeval(val_dataloader,model):
total_valloss = 0
model.eval()
with torch.no_grad() :
for val_x,val_y in val_dataloader:
output = model(val_x)
 valloss = criterion(output,val_y)
 total_valloss += valloss.item()
return total_valloss
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
all_trainloss = []
all_valloss = []
model=Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , lr=0.01)
 for epoch in tqdm(range(1000)) :
model,trainloss = training(train_dataloader,model)
 all_trainloss.append(trainloss)
 valloss = valeval(val_dataloader,model)
 all_valloss.append(valloss )
def prep_data(signal_ids, dataset="train"):
signal_ids_all = np.concatenate(signal_ids)
 if dataset == "train":
praq_data = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
elif dataset == "test":
praq_data = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
else:
raise ValueError("Unknown dataset")
 X = []
for sids in signal_ids:
data = praq_data[[str(s)for s in sids]].values.T
X_signal = [transform_ts(signal)for signal in data]
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 return X<prepare_x_and_y>
train_preds = model(train_X)
 train_preds[:3]
X, y = get_features("train", split_parts=6 )<choose_model_class>
train_preds2 = torch.max(train_preds.data,1)[1]
train_preds2[:3]
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
init_glorot_uniform = initializers.glorot_uniform(seed=RANDOM_SEED)
 init_orthogonal = initializers.orthogonal(seed=RANDOM_SEED)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu", kernel_initializer=init_glorot_uniform )(x)
 x = Dense(1, activation="sigmoid", kernel_initializer=init_glorot_uniform )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[StatefullMCC(np.linspace(0.45,0.55,11)) ])
 return model<split>
p_train["preds"] = train_preds2
p_train
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape<compute_test_metric>
p_train["judge"] = p_train["Survived"] == p_train["preds"]
p_train
def threshold_search(y_true, y_proba):
thresholds = np.linspace(0.0,1.0,101)
 scores = [matthews_corrcoef(y_true,(y_proba > t ).astype(np.uint8)) for t in thresholds]
best_idx = np.argmax(scores)
 return thresholds[best_idx], scores[best_idx]<compute_test_metric>
accuracy_score(train_Y,train_preds2 )
best_threshold, best_score = threshold_search(y_val, preds_val)
 print(best_threshold, best_score )<load_from_csv>
def calc_accuracy(x,y,model):
preds = model(x)
 preds2 = torch.max(preds.data,1)[1]
return accuracy_score(y,preds2)
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )<drop_column>
calc_accuracy(train_X,train_Y,model )
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<prepare_x_and_y>
calc_accuracy(val_X,val_Y,model )
%%time
X_test_input, _ = get_features("test" )<load_from_csv>
all_trainloss = []
all_valloss = []
all_trainscore = []
all_valscore = []
bestscore = None
model=Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , lr=0.01)
 for epoch in tqdm(range(1000)) :
model,trainloss = training(train_dataloader,model)
 all_trainloss.append(trainloss)
 valloss = valeval(val_dataloader,model)
 all_valloss.append(valloss)
 trainscore = calc_accuracy(train_X,train_Y,model)
 all_trainscore.append(trainscore)
 valscore = calc_accuracy(val_X,val_Y,model)
 all_valscore.append(valscore)
 if bestscore is None:
bestscore = valscore
state = {
'state_dict': model.state_dict() ,
'optimizer_dict': optimizer.state_dict() ,
"bestscore":bestscore
}
torch.save(state, "model1.pth")
 elif bestscore < valscore:
bestscore = valscore
state = {
'state_dict': model.state_dict() ,
'optimizer_dict': optimizer.state_dict() ,
"bestscore":bestscore
}
torch.save(state, "model1.pth")
 else:
pass
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
state = torch.load("./model1.pth" )
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
<data_type_conversions>
model.load_state_dict(state["state_dict"] )
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 submission
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
test_X = test[FEATURES]
test_X
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *<define_variables>
test_X = np.array(test[FEATURES])
 test_X = scaler.transform(test_X)
 test_X = torch.from_numpy(test_X ).float()
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
preds = model(test_X )
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<choose_model_class>
preds2 = torch.max(preds.data,1)[1]
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
submission["Survived"] = preds2
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<define_variables>
submission.to_csv("submission1.csv",index = False )
max_num = 127
min_num = -128<categorify>
bestscores = []
for fold in range(5):
trn_idx = folds[folds['fold'] != fold].index
val_idx = folds[folds['fold'] == fold].index
train_folds = folds.loc[trn_idx].reset_index(drop=True)
 valid_folds = folds.loc[val_idx].reset_index(drop=True)
 model=Net()
p_train = train_folds.copy()
p_val = valid_folds.copy()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , lr=0.01)
 train_X = np.array(p_train[FEATURES])
 train_Y = np.array(p_train[TARGET])
 val_X = np.array(p_val[FEATURES])
 val_Y = np.array(p_val[TARGET])
 scaler = StandardScaler()
scaler.fit(train_X)
 train_X = scaler.transform(train_X)
 val_X = scaler.transform(val_X)
 train_X = torch.from_numpy(train_X ).float()
train_Y = torch.from_numpy(train_Y ).long()
val_X = torch.from_numpy(val_X ).float()
val_Y = torch.from_numpy(val_Y ).long()
train_dataset = TensorDataset(train_X,train_Y)
 val_dataset = TensorDataset(val_X,val_Y)
 train_dataloader = DataLoader(train_dataset,batch_size=128,shuffle = True)
 val_dataloader = DataLoader(val_dataset,batch_size=128,shuffle = False)
 all_trainloss = []
all_valloss = []
all_trainscore = []
all_valscore = []
bestscore = None
model=Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , lr=0.01)
 for epoch in tqdm(range(1000)) :
model,trainloss = training(train_dataloader,model)
 all_trainloss.append(trainloss)
 valloss = valeval(val_dataloader,model)
 all_valloss.append(valloss)
 trainscore = calc_accuracy(train_X,train_Y,model)
 all_trainscore.append(trainscore)
 valscore = calc_accuracy(val_X,val_Y,model)
 all_valscore.append(valscore)
 if bestscore is None:
bestscore = valscore
state = {
'state_dict': model.state_dict() ,
'optimizer_dict': optimizer.state_dict() ,
"bestscore":bestscore
}
torch.save(state, "model"+str(fold)+ ".pth")
 elif bestscore < valscore:
bestscore = valscore
state = {
'state_dict': model.state_dict() ,
'optimizer_dict': optimizer.state_dict() ,
"bestscore":bestscore
}
torch.save(state, "model"+str(fold)+ ".pth")
 else:
pass
bestscores.append(bestscore)
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
states = [torch.load("model" + str(s)+ ".pth")for s in range(5)]
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
soft_values = []
for state in states:
model.load_state_dict(state["state_dict"])
 model.eval()
with torch.no_grad() :
y_pred = model(test_X)
 soft_values.append(y_pred.softmax(1 ).to("cpu" ).numpy())
 soft_values2 = np.mean(soft_values,axis=0 )
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<concatenate>
preds2 = [soft_values2[s].argmax() for s in range(len(soft_values2)) ]
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )<choose_model_class>
submission["Survived"] = preds2
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model<split>
submission.to_csv("submission2.csv",index=False )