Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
pd.options.mode.chained_assignment = None
knowledge_data = pd.read_csv('.. /input/vsb-knowledge-0744/sub_ens_v16.csv' )<load_from_csv>
test_data = pd.read_csv('.. /input/titanic/test.csv')
 train_data = pd.read_csv('.. /input/titanic/train.csv')
 comb = [train_data, test_data]
if 'v_raw_train' in globals() :
del v_raw_train
id_test = np.loadtxt('.. /input/vsb-power-line-fault-detection/metadata_test.csv', skiprows=1, delimiter=',')[:, 0].astype(int)
 n_test = len(id_test)
 X_tests = []
n_subset = 4
nread = 0
for i_subset in range(n_subset):
ibegin = 8712 + 3*int(n_test // 3 *(i_subset/n_subset))
iend = 8712 + 3*int(n_test // 3 *(( i_subset + 1)/n_subset))
print('Loading %d/%d; signal_id %d - %d...' %(i_subset, n_subset, ibegin, iend))
v_raw_test = pq.read_pandas('.. /input/vsb-power-line-fault-detection/test.parquet',
columns=[str(i)for i in range(ibegin, iend)] ).to_pandas().values
nread += v_raw_test.shape[1]
X = compute_features(v_raw_test)
 X_tests.append(X)
 print('%d/%d test data processed.' %(nread, n_test))
del v_raw_test
X_test = np.concatenate(X_tests, axis=0)
 assert(X_test.shape[0] == id_test.shape[0] // 3)
 del X_tests
print('X_test computation done.shape', X_test.shape )<concatenate>
for ds in comb:
age_avg = ds['Age'].mean()
age_std = ds['Age'].std()
age_nan = ds['Age'].isnull().sum()
age_rand_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_nan)
 ds['Age'][np.isnan(ds['Age'])] = age_rand_list
ds['Age'] = ds['Age'].astype(int )
len_train=len(X_all3)
 y_list=knowledge_data['target'].values
y_test=[]
for j in range(0,len(y_list),3):
y_test.append(y_list[j])
 y_test=np.asarray(y_test)
 del knowledge_data
print(X_all3.shape,y_all3.shape)
 print(X_test.shape,y_test.shape)
 X_all3=np.concatenate([X_all3,X_test])
 y_all3=np.concatenate([y_all3,y_test])
<train_model>
emb_mode = comb[0]['Embarked'].mode().at[0]
comb[0]['Embarked'] = comb[0]['Embarked'].fillna(emb_mode )
n_splits = 5
models = []
scores = np.zeros(n_splits)
 print('Training...')
 print('MCC training & cv')
 seeds=[0,42,1204,2019]
for seed in seeds:
splits = list(sklearn.model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed ).split(X_all3, y_all3))
for i,(idx_train, idx_cv)in enumerate(splits):
X_train = X_all3[idx_train, :]
y_train = y_all3[idx_train]
X_cv = X_all3[idx_cv, :]
y_cv = y_all3[idx_cv]
learning_rate = 0.006
model = CatBoostClassifier(learning_rate=learning_rate, od_type='IncToDec',
loss_function='Logloss', use_best_model=True, eval_metric='MCC')
 model.fit(X_train, y_train.astype(float), silent=True,
eval_set=(X_cv, y_cv.astype(float)))
 y_predict_train = model.predict(X_train)
 y_predict_cv = model.predict(X_cv)
 score_cv = sklearn.metrics.matthews_corrcoef(y_cv, y_predict_cv)
 models.append(model)
 scores[i] = score_cv
print('CV scores %.3f ± %.3f' %(np.mean(scores), np.std(scores)) )<load_from_csv>
fare = float(comb[1]['Fare'].mode())
 comb[1]['Fare'] = comb[1]['Fare'].fillna(fare )
<predict_on_test>
for ds in comb:
ds['Cabin_Null'] = ds['Cabin'].isnull().astype(int)
 pd.crosstab(comb[0]['Cabin_Null'], comb[0]['Survived'] )
y_test_probas = np.empty(( X_test.shape[0], n_splits*len(seeds)))
 for i, model in enumerate(models):
y_test_probas[:, i] = model.predict_proba(X_test)[:, 1]
y_test_proba = np.mean(y_test_probas, axis=1)
 y_submit = np.repeat(y_test_proba > 0.25, 3)
 print('Positive fraction %d/%d = %.3f' %(
 np.sum(y_submit), len(y_submit), np.sum(y_submit)/len(y_submit)) )<save_to_csv>
comb[0][['Cabin_Null', 'Survived']].groupby(['Cabin_Null'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
assert(len(id_test)== len(y_submit))
n = len(id_test)
 with open('submission.csv', 'w')as f:
f.write('signal_id,target
')
 for i in range(n):
f.write('%d,%d
' %(id_test[i], int(y_submit[i])))
 print('submission.csv written' )<import_modules>
for daset in comb:
daset['TickType'] = 0
tic_let = daset['Ticket'].str.extract(r'(^\S+)')
 daset['TickType'] = tic_let
daset['TickType'] = daset['TickType'].fillna('Other')
 comb[0]['TickType'].unique()
import os
import pandas as pd
import numpy as np
import pyarrow.parquet as pq
from keras.layers import *
from keras.callbacks import *
from keras.initializers import *
from keras import optimizers
from keras import backend as K
from keras.models import Model
import tensorflow as tf
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm<define_variables>
for dataset in comb:
for letter in "ACFPSW":
let_regex = r"^(" + letter + ".*)"
l_tic = dataset.TickType.str.extract(let_regex ).dropna(axis=0 ).reset_index(drop=True)
 tic_l_list = list(l_tic[0].unique())
 dataset['TickType'].replace(tic_l_list, letter, inplace=True)
 pd.crosstab(comb[0].TickType, comb[0].Survived )
sample_size = 800000<load_from_csv>
for ds in comb:
ds['TickType'] = ds['TickType'].replace('F', 'Other')
 comb[0][['TickType', 'Survived']].groupby(['TickType'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<normalization>
for ds in comb:
ds['RelatAboard'] = ds['SibSp'] + ds['Parch']
comb[0][['RelatAboard', 'Survived']].groupby(['RelatAboard'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
max_num = 127
min_num = -128
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
for ds in comb:
ds['OTTmembers'] = 2
ds.loc[ds['RelatAboard'] == 0, 'OTTmembers'] = 0
ds.loc[(ds['RelatAboard'] == 1)|(ds['RelatAboard'] == 2), 'OTTmembers'] = 1
ds.loc[ds['RelatAboard'] == 3, 'OTTmembers'] = 3
comb[0][['OTTmembers', 'Survived']].groupby(['OTTmembers'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
for ds in comb:
ds['Rank'] = 0
rank = ds['Name'].str.extract(r'(\w+\.) ')
 ds['Rank'] = rank
ranks = set(comb[0]['Rank'].values)
 print(ranks )
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<concatenate>
ot_list = ['Capt.', 'Col.', 'Countess.', 'Don.', 'Dr.', 'Jonkheer.', 'Rev.', 'Sir.', 'Major.']
miss_list = ['Lady.', 'Mlle.', 'Mme.', 'Ms.']
for dataset in comb:
dataset['Rank'].replace(ot_list, 'Other', inplace=True)
 dataset['Rank'].replace(miss_list, 'Miss.', inplace=True)
 comb[0][['Rank', 'Survived']].groupby(['Rank'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )<compute_test_metric>
comb[0]['AgeGroup'] = pd.cut(comb[0]['Age'], 5)
 comb[0][['AgeGroup', 'Survived']].groupby(['AgeGroup'], as_index=False ).mean().sort_values(by='AgeGroup', ascending=False )
def matthews_correlation(y_true, y_pred):
y_pred = tf.convert_to_tensor(y_pred, np.float32)
 y_true = tf.convert_to_tensor(y_true, np.float32)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<set_options>
for ds in comb:
ds.loc[ds['Age'] < 16, 'Age'] = 0
ds.loc[(ds['Age'] >= 16)&(ds['Age'] < 48), 'Age'] = 1
ds.loc[(ds['Age'] >= 48)&(ds['Age'] < 64), 'Age'] = 2
ds.loc[ds['Age'] >= 64, 'Age'] = 3
ds['Age'] = ds['Age'].astype(int)
 comb[0][['Age', 'Survived']].groupby(['Age'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<normalization>
for ds in comb:
ds.loc[ds['Fare'] <= 10.5, 'Fare'] = 0
ds.loc[(ds['Fare'] > 10.5)&(ds['Fare'] <= 39.688), 'Fare'] = 1
ds.loc[ds['Fare'] > 39.688, 'Fare'] = 2
ds['Fare'] = ds['Fare'].astype(int)
 comb[0][['Fare', 'Survived']].groupby(['Fare'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<choose_model_class>
s_dict = {'male': 0, 'female': 1}
for ds in comb:
ds['Sex'] = ds['Sex'].map(s_dict)
 ds['Embarked'] = ds['Embarked'].astype('category' ).cat.codes
ds['TickType'] = ds['TickType'].astype('category' ).cat.codes
ds['Rank'] = ds['Rank'].astype('category' ).cat.codes
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)) )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)) )(x)
 x_1 = Attention(input_shape[1] )(x)
 x_1 = Dropout(0.5 )(x_1)
 x_2 = Capsule(num_capsule=8, dim_capsule=8, routings=4, share_weights=True )(x)
 x_2 = Flatten()(x_2)
 x_2 = Dropout(0.5 )(x_2)
 x_rcnn = Conv1D(filters=128,
kernel_size=1,
kernel_initializer='he_uniform' )(inp)
 x_rcnn = Activation('relu' )(x_rcnn)
 x_rcnn_atten = Attention(input_shape[1] )(x_rcnn)
 x_rcnn_capsule = Capsule(num_capsule=8, dim_capsule=8, routings=4, share_weights=True )(x_rcnn)
 x_rcnn_capsule = Flatten()(x_rcnn_capsule)
 conc = concatenate([x_1, x_2, x_rcnn_atten, x_rcnn_capsule])
 conc = Dense(512, activation="relu" )(conc)
 conc = Dropout(0.5 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model<train_model>
drop_col_test = ['Cabin', 'Name', 'Parch', 'SibSp', 'RelatAboard', 'Ticket']
drop_col_train = drop_col_test.copy()
drop_col_train.extend(['FareGroup', 'AgeGroup', 'PassengerId'])
 comb[0] = comb[0].drop(columns=drop_col_train)
 comb[1] = comb[1].drop(columns=drop_col_test )
N_SPLITS = 5
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=1029 ).split(X, y))
preds_val = []
y_val = []
best_scores = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 history = model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 best_scores.append(np.max(history.history['val_matthews_correlation']))
model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 print("
" + str(best_scores))
print("
" + str(np.mean(best_scores)) )<concatenate>
train = comb[0]
test = comb[1]
preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 print(preds_val.shape, y_val.shape )<compute_test_metric>
train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result<compute_test_metric>
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
best_threshold = threshold_search(y_val, preds_val)['threshold']
print(best_threshold )<load_from_csv>
train[['Age', 'Survived']].groupby(['Age'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
meta_test = pd.read_csv('.. /input/metadata_test.csv')
 meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<define_variables>
train[['Fare', 'Survived']].groupby(['Fare'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )<load_from_csv>
train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
submission = pd.read_csv('.. /input/sample_submission.csv' )<concatenate>
train[['Cabin_Null', 'Survived']].groupby(['Cabin_Null'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 X_test_input.shape<predict_on_test>
train[['TickType', 'Survived']].groupby(['TickType'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )<data_type_conversions>
train[['OTTmembers', 'Survived']].groupby(['OTTmembers'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
train[['Rank', 'Survived']].groupby(['Rank'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head(10 )<count_values>
y_list = list(train['Survived'])
 y = pd.Series(y_list, name='Survived')
 X = train.drop(columns='Survived')
 X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2 )
submission.target.value_counts().sort_index()<import_modules>
random_forest = RandomForestClassifier(bootstrap=True, n_estimators=700, criterion='entropy')
 random_forest.fit(X_train, y_train)
 print("Accuracy on train data: ", random_forest.score(X_train, y_train))
print("Accuracy on test data: ", random_forest.score(X_test, y_test))
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import *
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
from keras import activations
from keras import regularizers
from keras import initializers
from keras import constraints
from sklearn.preprocessing import MinMaxScaler
from numba import jit
from math import log, floor
from sklearn.neighbors import KDTree
from scipy.signal import periodogram, welch
from keras.engine import Layer
from keras.engine import InputSpec
from keras.objectives import categorical_crossentropy
from keras.objectives import sparse_categorical_crossentropy 
<define_variables>
params={'n_neighbors' : range(1, 20), 'leaf_size' : range(1, 50)}
knn_grid = GridSearchCV(KNeighborsClassifier() , params, scoring='roc_auc')
 knn_grid.fit(X_train, y_train)
 print("Accuracy on train data: ", knn_grid.score(X_train, y_train))
print("Accuracy on test data: ", knn_grid.score(X_test, y_test))
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
params2 = {'alpha' : [0.00001, 0.0001, 0.001, 0.01, 1, 10, 100, 1000], 'normalize' : [True, False], 'random_state' : [0, 50, 100, 150, 200]}
r_grid = GridSearchCV(RidgeClassifier() , params2, scoring='roc_auc')
 r_grid.fit(X_train, y_train)
 print("Accuracy on train data: ", r_grid.score(X_train, y_train))
print("Accuracy on test data: ", r_grid.score(X_test, y_test))
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<choose_model_class>
svm_classifier = svm.SVC(kernel="poly", C=3, degree=6)
 svm_classifier.fit(X_train, y_train)
 print("Accuracy on train data: ", svm_classifier.score(X_train, y_train))
print("Accuracy on test data: ", svm_classifier.score(X_test, y_test))
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
gbm = xgb.XGBClassifier(
 learning_rate = 0.02,
use_label_encoder=False,
n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
eval_metric = 'error',
scale_pos_weight=1 ).fit(X_train, y_train)
 print("Accuracy on train data: ", gbm.score(X_train, y_train))
print("Accuracy on test data: ", gbm.score(X_test, y_test))
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<define_variables>
Y_train_full = train['Survived']
X_train_full = train.drop(columns='Survived')
 X_test_full = test.drop(columns='PassengerId')
 X_train_full.shape, X_test_full.shape, Y_train_full.shape
max_num = 127
min_num = -128<categorify>
model = svm.SVC(kernel="poly", C=2)
 model.fit(X_train_full, Y_train_full)
 print("Accuracy on train data: ", model.score(X_train_full, Y_train_full))
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
y_test_predicted = model.predict(X_test_full)
 y_test_predicted
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
sub = pd.DataFrame({
"PassengerId": comb[1]['PassengerId'],
"Survived": y_test_predicted
})
 sub.to_csv('submission.csv', index = False )