Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<predict_on_test>
import csv
import re
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.python.framework import ops
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
def predict(Xp, catp, thres):
pred = np.zeros(len(Xp))
for i in range(6):
bins = [-99.0] + list(np.sort(thres[i])) + [99.0]
pred[catp==i] = np.digitize(Xp[catp==i], bins)-1
return pred
def calculate_matrix(transition_matrix, states, number_processes):

 for i in range(transition_matrix.shape[0]):
transition_matrix[i, i] = 1 - np.sum(transition_matrix[i, :])
 n0 = len(states)
 new_transition_matrix = transition_matrix.copy()
new_states = [(x,)for x in range(n0)]
for process in range(1, number_processes):
nc = new_transition_matrix.shape[0]
temp_transition_matrix = np.zeros(( n0*nc, n0*nc))
temp_states = []
for i in range(n0):
temp_states.extend([s +(i,)for s in new_states])
 for j in range(n0):
temp_transition_matrix[i*nc:(i+1)*nc, j*nc:(j+1)*nc] = transition_matrix[i][j] * new_transition_matrix
new_states = sorted(list(set([tuple(sorted(x)) for x in temp_states])))
 new_transition_matrix = np.zeros(( len(new_states), len(new_states)))
 for i in range(len(new_states)) :
ix_i = [k for k, x in enumerate(temp_states)if tuple(sorted(x)) == new_states[i]]
for j in range(len(new_states)) :
ix_j = [k for k, x in enumerate(temp_states)if tuple(sorted(x)) == new_states[j]]
new_transition_matrix[i, j] = np.sum(temp_transition_matrix[ix_i, :][:, ix_j])
 new_transition_matrix[i, j] /= len(ix_i)
 new_channels = []
for s in new_states:
new_channels.append(sum([states[x] for x in s]))
new_channels= np.array(new_channels)
 return new_transition_matrix, new_channels
def create_hmm(signal, predictions, transmat, states):
means = np.ones(( len(states)))* np.NaN
for c in range(np.min(predictions), np.max(predictions)+ 1):
mu = np.nanmedian(signal[predictions == c])
 ix = np.where(states == c)[0]
for i in ix:
means[i] = mu
mask = ~np.isnan(means)
 lr = LinearRegression()
lr.fit(states[mask].reshape(-1, 1), means[mask])
 means = lr.predict(states.reshape(-1, 1))
hmm = GaussianHMM(
 n_components=len(states),
n_iter=50,
verbose=False,
algorithm='map',
params='stmc',
random_state=42,
init_params='s',
covariance_type='tied',
tol=0.01 
)
 hmm.n_features = 1
hmm.means_ = means.reshape(-1, 1)
 covs = np.array([np.cov(signal[~np.isnan(signal)][predictions[~np.isnan(signal)] == c])for c in states if np.sum(~np.isnan(signal)== c)> 1000])
 hmm.covars_ = np.nanmean(covs ).reshape(-1, 1)
 hmm.transmat_ = transmat
return hmm<concatenate>
def load_data(train_path, test_path):

 train_data = pd.read_csv(train_path)
 test_data = pd.read_csv(test_path)
 print("number of training examples = " + str(train_data.shape[0]))
print("number of test examples = " + str(test_data.shape[0]))
print("train shape: " + str(train_data.shape))
print("test shape: " + str(test_data.shape))
return train_data, test_data
def pre_process_data(df):

 df = df.set_index("PassengerId")
 df['gender'] = df['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 df.loc[df.Embarked.isnull() , 'Embarked'] = df['Embarked'].mode() [0]
for gender in df['gender'].unique() :
median_age = df[(df['gender'] == gender)].Age.median()
df.loc[(df['Age'].isnull())&(df['gender'] == gender), 'Age'] = median_age
df = pd.get_dummies(df, columns=['gender'])
 df = pd.get_dummies(df, columns=['Embarked'])
 bins = [0, 4, 12, 18, 30, 50, 65, 100]
age_index =(1, 2, 3, 4, 5, 6, 7)
 df['Age-bin'] = pd.cut(df['Age'], bins, labels=age_index ).astype(int)
 df['family'] = df['SibSp'] + df['Parch'] + 1
df['family'] = df['family'].map(lambda x: 4 if x > 4 else x)
 df['FTicket'] = df['Ticket'].map(lambda x: x[0])
 df['FTicket'] = df['FTicket'].replace(['W', 'F', 'L', '5', '6', '7', '8', '9'], '4')
 df = pd.get_dummies(df, columns=['FTicket'])
 df['title'] = df.apply(lambda row: re.split('[,.]+', row['Name'])[1], axis=1)
 df['title'] = df['title'].map({' Capt': 'Other', ' Master': 'Master', ' Mr': 'Mr', ' Don': 'Other',
' Dona': 'Other', ' Lady': 'Other', ' Col': 'Other', ' Miss': 'Miss',
' the Countess': 'Other', ' Dr': 'Other', ' Jonkheer': 'Other', ' Mlle': 'Other',
' Sir': 'Other', ' Rev': 'Other', ' Ms': 'Other', ' Mme': 'Other', ' Major': 'Other',
' Mrs': 'Mrs'})
 df = pd.get_dummies(df, columns=['title'])
 df = df.drop(['Name', 'Ticket', 'Cabin', 'Sex', 'Fare', 'Age'], axis=1)
 return df
def mini_batches(train_set, train_labels, mini_batch_size):

 set_size = train_set.shape[0]
batches = []
num_complete_minibatches = set_size // mini_batch_size
for k in range(0, num_complete_minibatches):
mini_batch_x = train_set[k * mini_batch_size:(k + 1)* mini_batch_size]
mini_batch_y = train_labels[k * mini_batch_size:(k + 1)* mini_batch_size]
mini_batch =(mini_batch_x, mini_batch_y)
 batches.append(mini_batch)
 if set_size % mini_batch_size != 0:
mini_batch_x = train_set[(set_size -(set_size % mini_batch_size)) :]
mini_batch_y = train_labels[(set_size -(set_size % mini_batch_size)) :]
mini_batch =(mini_batch_x, mini_batch_y)
 batches.append(mini_batch)
 return batches
def get_Ptran_cat(cat):
if cat==0:
mat = [[0 , 0.1713 , 0 , 0 ],
[0.3297, 0 , 0 , 0.01381],
[0 , 1 , 0 , 0 ],
[0 , 0.0002686, 0 , 0 ]]
elif cat==1:
mat = [[0 , 0.0121, 0 , 0 ],
[0.0424, 0 , 0.2766, 0.0101],
[0 , 0.2588, 0 , 0 ],
[0 , 0.0239, 0 , 0 ]]
elif cat<=4:
mat = [[0 , 0.0067, 0 , 0 ],
[0.0373, 0 , 0.2762, 0.0230],
[0 , 0.1991, 0 , 0 ],
[0 , 0.0050, 0 , 0 ]]
elif cat==5:
EPS = 0
mat = [[0., EPS , 0.34493706, 0.00287762, 0.00006045, EPS ],
[EPS , 0., 0.00040108, EPS , EPS , EPS ],
[0.16435428, 0.00438756, 0., 0.01714043, 0.00023227, EPS ],
[0.02920171, 0.00080145, 0.27065939, 0., 0.01805161, 0.00108684],
[0.00268151, 0.00000064, 0.06197474, 0.30666751, 0., 0.06625158],
[EPS , EPS , 0.00000136, 0.13616454, 0.51059444, EPS ]]
return np.array(mat)
 def get_hidden_states(cat):
if cat!=5: return [1, 1, 0, 0]
else: return [0, 0, 1, 2, 3, 4]
def get_Psig(signal, States, kexp):
Psig = np.zeros(( len(signal), len(States)))
 for i in range(len(Psig)) :
Psig[i] = np.exp(( -(signal[i] - States)**2)/(kexp))
return Psig
def forward(Psig, Ptran, etat_in=None, coef=1, normalize=True):
if etat_in is None: etat_in = np.ones(Psig.shape)/Psig.shape[1]
alpha = np.zeros(Psig.shape)
 etat = np.zeros(Psig.shape)
 C = np.zeros(Psig.shape[0])
 etat[0] = etat_in[0]
alpha[0] = etat_in[0]
if normalize:
alpha[0] = etat_in[0]*Psig[0]
alpha[0]/=alpha[0].sum()
for j in range(1, Psig.shape[0]):
etat[j] = alpha[j-1]@Ptran
if normalize: etat[j] /= etat[j].sum()
etat[j] =(etat[j]**coef)*(( etat_in[j])**(1-coef))
if normalize: etat[j] /= etat[j].sum()
alpha[j] = etat[j] * Psig[j]
alpha[j] /= alpha[j].sum()
return alpha, etat
def optimize_thres_unsupervised(pred):
y = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv' ).open_channels.values
sig = np.load(PATH+'M301_sig.npy')
 catbatches = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 2, 3, 5, 1, 4, 3, 4, 5, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]
good_id = list(range(3600000)) +list(range(3900000,5000000))
sY_all = [[0.26]*12,
[0.26]*12,
[0.26]*12,
[0.22]*12,
[0.26]*12,
[0.26]*12,
]
L = 100000
Y = pred.copy()
Thres = {}
Yopt = pred.copy()
for b in range(70):
Thres[b] = np.zeros(12)
 Thres[b][0] = -99
Thres[b][-1] = 99
poscat = range(L*b, L*(b+1))
catbatch = int(catbatches[b])
 sY = sY_all[catbatch]
Yloc = Y[poscat]
floc = sig[poscat]
adaptive_sY = np.array([sY[int(np.round(item)) ] for item in floc])
 floc2 = floc[np.abs(floc-np.round(floc)) - adaptive_sY < 0]
for i in range(10):
ni = len(floc2[np.round(floc2)<=i])
 ni2 = np.round(ni*len(floc)/ max(1, len(floc2)) ).astype(int)
 Ys = np.concatenate([np.sort(floc), [19]])
 Thres[b][i+1] = 0.5*(Ys[max(0,ni2)]+Ys[min(len(Ys)-1,ni2)])
 for i in range(11):
Yloc[(Yloc>=Thres[b][i])&(Yloc<Thres[b][i+1])] = i
Yopt[poscat] = Yloc
print(f1_score(y[good_id], Yopt[:5000000][good_id], average='macro'))
return Yopt, Thres<define_variables>
def create_placeholders(input_size, output_size):

 x = tf.placeholder(shape=(None, input_size), dtype=tf.float32, name="X")
 y = tf.placeholder(shape=(None, output_size), dtype=tf.float32, name="Y")
 return x, y
def forward_propagation(x, parameters, keep_prob=1.0, hidden_activation='relu'):

 a_dropout = x
n_layers = len(parameters)// 2
for l in range(1, n_layers):
a_prev = a_dropout
a_dropout = linear_activation_forward(a_prev, parameters['w%s' % l], parameters['b%s' % l], hidden_activation)
 if keep_prob < 1.0:
a_dropout = tf.nn.dropout(a_dropout, keep_prob)
 al = tf.matmul(a_dropout, parameters['w%s' % n_layers])+ parameters['b%s' % n_layers]
return al
def linear_activation_forward(a_prev, w, b, activation):

 a = None
if activation == "sigmoid":
z = tf.matmul(a_prev, w)+ b
a = tf.nn.sigmoid(z)
 elif activation == "relu":
z = tf.matmul(a_prev, w)+ b
a = tf.nn.relu(z)
 elif activation == "leaky relu":
z = tf.matmul(a_prev, w)+ b
a = tf.nn.leaky_relu(z)
 return a
def initialize_parameters(layer_dims):

 parameters = {}
n_layers = len(layer_dims)
 for l in range(1, n_layers):
parameters['w' + str(l)] = tf.get_variable('w' + str(l), [layer_dims[l - 1], layer_dims[l]],
initializer=tf.contrib.layers.xavier_initializer())
 parameters['b' + str(l)] = tf.get_variable('b' + str(l), [layer_dims[l]], initializer=tf.zeros_initializer())
 return parameters
def compute_cost(z3, y):

 cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=z3, labels=y))
return cost
def predict(data, parameters):

 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 dataset = tf.cast(tf.constant(data), tf.float32)
 fw_prop_result = forward_propagation(dataset, parameters)
 fw_prop_activation = tf.nn.softmax(fw_prop_result)
 prediction = fw_prop_activation.eval()
return prediction
def accuracy(predictions, labels):

 prediction_size = predictions.shape[0]
prediction_accuracy = np.sum(np.argmax(predictions, 1)== np.argmax(labels, 1)) / prediction_size
return 100 * prediction_accuracy
def minibatch_accuracy(predictions, labels):

 prediction_size = predictions.shape[0]
prediction_accuracy = np.sum(np.argmax(predictions, 1)== np.argmax(labels, 1)) / prediction_size
return 100 * prediction_accuracy
def l2_regularizer(cost, l2_beta, parameters, n_layers):

 regularizer = 0
for i in range(1, n_layers):
regularizer += tf.nn.l2_loss(parameters['w%s' % i])
 cost = tf.reduce_mean(cost + l2_beta * regularizer)
 return cost
def build_submission_name(layers_dims, num_epochs, lr_decay,
learning_rate, l2_beta, keep_prob, minibatch_size, num_examples):

 submission_name = 'ly{}-epoch{}.csv' \ 
.format(layers_dims, num_epochs)
 if lr_decay != 0:
submission_name = 'lrdc{}-'.format(lr_decay)+ submission_name
else:
submission_name = 'lr{}-'.format(learning_rate)+ submission_name
if l2_beta > 0:
submission_name = 'l2{}-'.format(l2_beta)+ submission_name
if keep_prob < 1:
submission_name = 'dk{}-'.format(keep_prob)+ submission_name
if minibatch_size != num_examples:
submission_name = 'mb{}-'.format(minibatch_size)+ submission_name
return submission_name
def plot_model_cost(train_costs, validation_costs, submission_name):

 plt.plot(np.squeeze(train_costs), label='Train cost')
 plt.plot(np.squeeze(validation_costs), label='Validation cost')
 plt.ylabel('cost')
 plt.xlabel('iterations(per tens)')
 plt.title("Model: " + submission_name)
 plt.legend()
plt.show()
def plot_model_accuracy(train_accuracies, validation_accuracies, submission_name):

 plt.plot(np.squeeze(train_accuracies), label='Train accuracy')
 plt.plot(np.squeeze(validation_accuracies), label='Validation accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('iterations(per tens)')
 plt.title("Model: " + submission_name)
 plt.legend()
plt.show()
PATH = '/kaggle/input/ion-cleaned-data/'
Kexp = [.103,.120,.1307,.138,.267,.105]
Kexpp = [1.8, 1.8, 1.8, 1.83, 1.807, 1.8]
N_PROCESSES = [1, 1, 3, 5, 10, 1]
COEFS_BACK = [1,.9192,.9192,.8792,.9022,.9192]
COEFS_FOR = [1,.8869,.8869,.8869,.8849,.8869]
COEFS_FIN = [.618, 0.50, 0.50, 0.49, 0.509, 0.50]
COEFS_FIN3 = [0.3, 0.3, 0.3, 0.35, 0.335, 0.3]
BATCHES = np.array([0, 5, 6, 10, 15, 20, 25, 30, 35, 40, 45, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 65])
 CATEGORIES = np.array([1, 1, 1, 2, 3, 5, 4, 2, 3, 4, 5, 6, 3, 4, 6, 2, 5, 4, 5, 6, 3, 6, 6])
 full_pred = np.zeros(7000000)
 cleaned_signal = np.load(PATH+'sig_301_sinecleaned.npy')
 oofs = pd.read_csv('.. /input/ion-oofs/YM257method2oofpredLB9446Proba/YM257method2oofpredLB9446Proba.txt',
index_col=None, skiprows=1, header=None, sep=' ' ).values
predictions = np.argmax(oofs, axis=1)
 for c in [0,1,2,3,4,5]:
print("
Training cat", c)
 kexp = Kexp[c]
kexpp = Kexpp[c]
coefback = COEFS_BACK[c]
coeffor = COEFS_FOR[c]
coef_fin = COEFS_FIN[c]
coef_fin3 = COEFS_FIN3[c]
Ptran, States = calculate_matrix(get_Ptran_cat(c), get_hidden_states(c), N_PROCESSES[c])
 for jb, b in enumerate(BATCHES):
if CATEGORIES[jb]!=c+1: continue
end_b = BATCHES[jb+1] if b!=65 else 70
sig = cleaned_signal[100000*b:100000*end_b]
nstates = Ptran.shape[0]
Psig = get_Psig(sig, States, kexp)
 if c!=5:
alpha0, etat0 = forward(Psig, Ptran, normalize=False)
 alpha1, etat1 = forward(Psig[::-1], np.transpose(Ptran), etat_in=etat0[::-1], coef=coefback)
 alpha2, etat2 = forward(Psig, Ptran, etat_in=etat1[::-1], coef=coeffor)
 alpha3 = etat1[::-1]*etat2*Psig**kexpp
for j, alp in enumerate(alpha3): alpha3[j] /= alp.sum()
pred = coef_fin*(alpha1[::-1])+(1-coef_fin-coef_fin3)*alpha2 + coef_fin3*alpha3
full_pred[b*100000:b*100000+len(sig)] = pred@States
print('Max/min',(pred@States ).max() ,(pred@States ).min())
 else:
for k in range(len(sig)// 100000):
sub_signal = sig[k*100000:(k+1)*100000]
oof = predictions[(b+k)*100000:(b+k+1)*100000]
hmm = create_hmm(sub_signal, oof, Ptran, States)
 hmm.fit(sub_signal.reshape(-1, 1))
pred = hmm.predict_proba(sub_signal.reshape(-1, 1))
full_pred[(b+k)*100000:(b+k+1)*100000] = pred @ States
print('Max/min',(pred@States ).max() ,(pred@States ).min())
 Yopt, Thres = optimize_thres_unsupervised(full_pred )<load_from_csv>
def model(train_set, train_labels, validation_set, validation_labels, layers_dims, learning_rate=0.01, num_epochs=1001,
print_cost=True, plot_cost=True, l2_beta=0., keep_prob=1.0, hidden_activation='relu', return_best=False,
minibatch_size=0, lr_decay=0, print_accuracy=True, plot_accuracy=True):

 ops.reset_default_graph()
input_size = layers_dims[0]
output_size = layers_dims[-1]
num_examples = train_set.shape[0]
n_layers = len(layers_dims)
 train_costs = []
validation_costs = []
train_accuracies = []
validation_accuracies = []
prediction = []
best_iteration = [float('inf'), 0, float('-inf'), 0]
best_params = None
if minibatch_size == 0 or minibatch_size > num_examples:
minibatch_size = num_examples
num_minibatches = num_examples // minibatch_size
if num_minibatches == 0:
num_minibatches = 1
submission_name = build_submission_name(layers_dims, num_epochs, lr_decay, learning_rate, l2_beta, keep_prob,
minibatch_size, num_examples)
 x, y = create_placeholders(input_size, output_size)
 tf_valid_dataset = tf.cast(tf.constant(validation_set), tf.float32)
 parameters = initialize_parameters(layers_dims)
 fw_output = forward_propagation(x, parameters, keep_prob, hidden_activation)
 train_cost = compute_cost(fw_output, y)
 train_prediction = tf.nn.softmax(fw_output)
 fw_output_valid = forward_propagation(tf_valid_dataset, parameters, 1, hidden_activation)
 validation_cost = compute_cost(fw_output_valid, validation_labels)
 valid_prediction = tf.nn.softmax(fw_output_valid)
 if l2_beta > 0:
train_cost = l2_regularizer(train_cost, l2_beta, parameters, n_layers)
 validation_cost = l2_regularizer(validation_cost, l2_beta, parameters, n_layers)
 if lr_decay != 0:
global_step = tf.Variable(0, trainable=False)
 learning_rate = tf.train.inverse_time_decay(learning_rate, global_step=global_step, decay_rate=lr_decay,
decay_steps=1)
 optimizer = tf.train.AdamOptimizer(learning_rate ).minimize(train_cost, global_step=global_step)
 else:
optimizer = tf.train.AdamOptimizer(learning_rate ).minimize(train_cost)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 for epoch in range(num_epochs):
train_epoch_cost = 0.
 validation_epoch_cost = 0.
 minibatches = mini_batches(train_set, train_labels, minibatch_size)
 for minibatch in minibatches: 
(minibatch_X, minibatch_Y)= minibatch
feed_dict = {x: minibatch_X, y: minibatch_Y}
_, minibatch_train_cost, prediction, minibatch_validation_cost = sess.run(
 [optimizer, train_cost, train_prediction, validation_cost], feed_dict=feed_dict)
 train_epoch_cost += minibatch_train_cost / num_minibatches
validation_epoch_cost += minibatch_validation_cost / num_minibatches
validation_accuracy = accuracy(valid_prediction.eval() , validation_labels)
 train_accuracy = accuracy(prediction, minibatch_Y)
 if print_cost is True and epoch % 500 == 0:
print("Train cost after epoch %i: %f" %(epoch, train_epoch_cost))
print("Validation cost after epoch %i: %f" %(epoch, validation_epoch_cost))
if print_accuracy is True and epoch % 500 == 0:
print('Train accuracy after epoch {}: {:.2f}'.format(epoch, train_accuracy))
print('Validation accuracy after epoch {}: {:.2f}'.format(epoch, validation_accuracy))
if plot_cost is True and epoch % 10 == 0:
train_costs.append(train_epoch_cost)
 validation_costs.append(validation_epoch_cost)
 if plot_accuracy is True and epoch % 10 == 0:
train_accuracies.append(train_accuracy)
 validation_accuracies.append(validation_accuracy)
 if return_best is True:
if validation_epoch_cost < best_iteration[0]:
best_iteration[0] = validation_epoch_cost
best_iteration[1] = epoch
best_params = sess.run(parameters)
 if validation_accuracy > best_iteration[2]:
best_iteration[2] = validation_accuracy
best_iteration[3] = epoch
best_params = sess.run(parameters)
 if return_best is True:
parameters = best_params
else:
parameters = sess.run(parameters)
 print("Parameters have been trained, getting metrics...")
 train_accuracy = accuracy(predict(train_set, parameters), train_labels)
 validation_accuracy = accuracy(predict(validation_set, parameters), validation_labels)
 print('Train accuracy: {:.2f}'.format(train_accuracy))
print('Validation accuracy: {:.2f}'.format(validation_accuracy))
print('Lowest validation cost: {:.2f} at epoch {}'.format(best_iteration[0], best_iteration[1]))
print('Highest validation accuracy: {:.2f} at epoch {}'.format(best_iteration[2], best_iteration[3]))
submission_name = 'tr_acc-{:.2f}-vd_acc{:.2f}-'.format(train_accuracy, validation_accuracy)+ submission_name
if return_best is True:
print('Lowest cost: {:.2f} at epoch {}'.format(best_iteration[0], best_iteration[1]))
if plot_cost is True:
plot_model_cost(train_costs, validation_costs, submission_name)
 if plot_accuracy is True:
plot_model_accuracy(train_accuracies, validation_accuracies, submission_name)
 return parameters, submission_name
Yopt_test = Yopt[5000000:]
sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv')
 sub['open_channels'] = Yopt_test.astype(np.int8)
 sub.to_csv('M318_Kha_withNewCat6.csv', index=None, float_format='%0.4f')
 plt.hist(Yopt[5000000:] )<install_modules>
TRAIN_PATH = '.. /input/train.csv'
TEST_PATH = '.. /input/test.csv'
train, test = load_data(TRAIN_PATH, TEST_PATH)
 CLASSES = 2
train_dataset_size = train.shape[0]
train_raw_labels = pd.get_dummies(train.Survived ).as_matrix()
!!pip install tensorflow_addons==0.9.1<import_modules>
train = pre_process_data(train)
 test = pre_process_data(test)
 train_pre = train.drop(['Survived'], axis=1 ).as_matrix().astype(np.float)
 test_pre = test.as_matrix().astype(np.float )
import tensorflow_addons as tfa<set_options>
standard_scaler = preprocessing.StandardScaler()
train_pre = standard_scaler.fit_transform(train_pre)
 test_pre = standard_scaler.fit_transform(test_pre)
 X_train, X_valid, Y_train, Y_valid = train_test_split(train_pre, train_raw_labels, test_size=0.3, random_state=1 )
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 FOLD = 4
AUG_CNT = 15
EPOCHS = 180
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.001
SPLITS = 5
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def read_data() :
train = pd.read_csv('.. /input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('.. /input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})

return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df['signal_2'] = df['signal']** 2
return df
def train_update_groups(df):
df['group_ind'] = -1
step_size = 500_000
groups = [0, 0, 1, 2, 4, 3, 1, 2, 3, 4]
for i in range(df.shape[0]//step_size):
a = i * step_size
b =(i+1)* step_size
df.loc[df.index[a:b], 'group_ind'] = groups[i]
df['group_ind'] = df['group_ind'].astype(int)
 return df
def test_update_groups(df):
df['group_ind'] = -1
step_size = 100_000
groups = [0, 2, 3, 0, 1, 4, 3, 4, 0, 2,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
for i in range(df.shape[0]//step_size):
a = i * step_size
b =(i+1)* step_size
df.loc[df.index[a:b], 'group_ind'] = groups[i]
df['group_ind'] = df['group_ind'].astype(int)
 return df
def convert_signal_into_probs(df, is_train=True, fit_dict=None):
if is_train:
df['group_ind'] = train_update_groups(df.copy())['group']
filt =(( df.time>=364.229)&(df.time<382.343)) |(( df.time>=47.857)&(df.time<47.863))
keep_filt= ~filt
fit_dict = {}
for gp in df['group_ind'].unique() :
x = df.loc[(df['group_ind'] == gp)& keep_filt, 'signal'].values
y = df.loc[(df['group_ind'] == gp)& keep_filt, 'open_channels'].values
fit_dict[gp] = {}
fit_dict[gp]['open_channels'] = y.max() +1
fit_dict[gp]['per_oc_dist'] = {}
for s in np.arange(fit_dict[gp]['open_channels']):
filt = y==s
mean_, std_ = np.mean(x[filt]), np.std(x[filt])
 fit_dict[gp]['per_oc_dist'][s] =(mean_, std_)
 print("GP = {:2d}, OC = {:2d}, Mean={:.4f}, Std={:.4f}".format(gp, s, mean_, std_))
else:
assert fit_dict is not None, "fit_dict should be provided when is_train is False"
df['group_ind'] = test_update_groups(df.copy())['group']
p_signal = np.zeros(( df.shape[0], 11))
for gp in df['group_ind'].unique() :
filt = df['group_ind'] == gp
x = df.loc[filt, 'signal'].values
for s in range(fit_dict[gp]['open_channels']):
dist = fit_dict[gp]['per_oc_dist'][s]
p_signal[filt, s] = norm.cdf(x, *dist)
 p_signal[filt, s] = np.where(p_signal[filt, s]>0.5, 1-p_signal[filt, s], p_signal[filt, s])
 p_signal_sum = p_signal.sum(axis=1)
 zero_sum_filt = p_signal_sum==0
filt_1 =(filt)&(~zero_sum_filt); p_signal[filt_1, :] = p_signal[filt_1, :] / p_signal_sum[filt_1].reshape(( -1,1))
filt_2 =(filt)&(zero_sum_filt); p_signal[filt_2, :] = 1./(fit_dict[gp]['open_channels']-1)
 df.drop(['signal', 'group_ind'], axis=1, inplace=True)
 for s in range(11):
df[f'sig_prob_{s}'] = p_signal[:, s]
return df, fit_dict
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time', 'group_ind']]
print(features)
 train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean ).astype(np.float32)
 test[feature] = test[feature].fillna(feature_mean ).astype(np.float32)
 return train, test, features
def Classifier_Wavenet_Gru(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters, kernel_size=1, dilation_rate=1, strides=1, padding="same" )(x)
 res_x = x
for dilation_rate in dilation_rates:
x = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'causal',
dilation_rate = dilation_rate )(x)
 tanh_out = Activation('tanh' )(x)
 sigm_out = Activation('sigmoid' )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = SpatialDropout1D(0.1 )(x)
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 x = Activation("relu" )(x)
 return res_x
def bidirectional_wave_block(x, filters, kernel_size, n):
x_f = wave_block(x, filters, kernel_size, n)
 x_rev = Lambda(lambda t: t[:,::-1,:] )(x)
 x_b = wave_block(x_rev, filters, kernel_size, n)
 x_b = Lambda(lambda t: t[:,::-1,:] )(x_b)
 x = concatenate([x_f, x_b])
 return x
inp = Input(shape =(shape_))
x = inp
x_rev = Lambda(lambda t: t[:,::-1,:] )(x)
 x = wave_block(x, 64, 2, 13)
 x = wave_block(x, 64, 2, 13)
 x = wave_block(x, 64, 2, 13)
 x_rev = wave_block(x_rev, 64, 2, 13)
 x_rev = wave_block(x_rev, 64, 2, 13)
 x_rev = wave_block(x_rev, 64, 2, 13)
 x_rev = Lambda(lambda t: t[:,::-1,:] )(x_rev)
 x = concatenate([x, x_rev])
 x = Bidirectional(tf.compat.v1.keras.layers.CuDNNGRU(64, return_sequences=True))(x)
 x = Bidirectional(tf.compat.v1.keras.layers.CuDNNGRU(64, return_sequences=True))(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def Classifier_Wavenet(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters, kernel_size=1, dilation_rate=1, strides=1, padding="same" )(x)
 res_x = x
for dilation_rate in dilation_rates:
x = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
dilation_rate = dilation_rate )(x)
 tanh_out = Activation('tanh' )(x)
 sigm_out = Activation('sigmoid' )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = SpatialDropout1D(0.1 )(x)
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 x = Activation("relu" )(x)
 return res_x
inp = Input(shape =(shape_))
x = inp
x = wave_block(x, 64, 3, 12)
 x = wave_block(x, 64, 3, 8)
 x = wave_block(x, 64, 3, 4)
 x = wave_block(x, 64, 3, 1)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt, clipvalue=1.0)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def do_augment(train, feats, trn_idx, aug_time=5):
aug_gp_mapping = {
3: 1,
4: 3
}
all_dfs = []
for gp in [0,1,2,3,4]:
orig_trn = train.loc[trn_idx, ['signal', 'group_ind', 'open_channels', 'group']]
orig_trn = orig_trn.loc[orig_trn.group_ind == gp]
orig_trn_x = orig_trn['signal'].values
orig_trn_y = orig_trn['open_channels'].values
print(orig_trn_x.shape, orig_trn_y.shape)
 print(orig_trn['open_channels'].value_counts().sort_index())
 means_ = []
vars_ = []
for oc in range(orig_trn_y.max() +1):
filt = orig_trn_y == oc
signals = orig_trn_x[filt]
means_ += [signals.mean() ]
vars_ += [signals.var() ]
print(oc, signals.mean() , signals.var())
 if gp not in aug_gp_mapping.keys() or aug_time == 0:
res = pd.DataFrame()
res['signal'] = orig_trn_x
res['open_channels'] = orig_trn_y
all_dfs += [res]
continue
trn = train.loc[trn_idx, ['signal', 'group_ind', 'open_channels', 'group']]
trn = trn.loc[trn.group_ind == aug_gp_mapping[gp],]
trn_x = trn['signal'].values
trn_y = trn['open_channels'].values
signal_segments = trn.groupby('group' ).apply(lambda x: x['signal'].values)
 open_chnannel_segments = trn.groupby('group' ).apply(lambda x: x['open_channels'].values)
 signal_segments = [s for s in signal_segments if len(s)== GROUP_BATCH_SIZE]
open_chnannel_segments = [s for s in open_chnannel_segments if len(s)== GROUP_BATCH_SIZE]
plt.title('before augment signal'); plt.plot(orig_trn_x[::1000]); plt.show()
plt.title('before augment label'); plt.plot(orig_trn_y[::1000]); plt.show()
seq_len = orig_trn_y.max() +1
selected_len = int(seq_len * 2 / 3)
 lr = LinearRegression()
lr.fit(np.arange(seq_len-selected_len, seq_len, 1 ).reshape(( -1,1)) , means_[-selected_len:])
 new_means = lr.predict(np.arange(seq_len ).reshape(( -1,1)))
 new_vars = [v if(orig_trn_y == i ).sum() >30 else max(vars_)for i, v in enumerate(vars_)]
for oc in range(orig_trn_y.max() +1):
filt = orig_trn_y == oc
signals = orig_trn_x[filt]
src_m, src_v = signals.mean() , signals.var()
if(orig_trn_y == oc ).sum() <=30:
src_v = max(vars_)
 tar_m, tar_v = new_means[oc], new_vars[oc]
signals =(signals-src_m)/(src_v**.5)*(tar_v**.5)+ tar_m
orig_trn_x[orig_trn_y == oc] = signals
aug_size = aug_time*orig_trn_x.shape[0]//GROUP_BATCH_SIZE
selections = [np.random.choice(np.arange(len(signal_segments)) , orig_trn_y.max() //trn['open_channels'].max() , replace=False)for _ in range(aug_size)]
combined_signal_segments = []
combined_oc_segments = []
for inds in selections:
combined_signal_segments += [np.vstack([signal_segments[ix] for ix in inds] ).sum(axis=0)]
combined_oc_segments += [np.vstack([open_chnannel_segments[ix] for ix in inds] ).sum(axis=0)]
aug_trn_x = np.concatenate(combined_signal_segments)
 aug_trn_y = np.concatenate(combined_oc_segments)
 print(pd.Series(aug_trn_y ).value_counts().sort_index())
 for oc in range(orig_trn_y.max() +1):
aug_signals = aug_trn_x[aug_trn_y == oc]
src_m, src_v = aug_signals.mean() , aug_signals.var()
if(aug_trn_y == oc ).sum() <=30:
src_v = max(vars_)
 tar_m, tar_v = new_means[oc], new_vars[oc]
aug_signals =(aug_signals-src_m)/(src_v**.5)*(tar_v**.5)+ tar_m
aug_trn_x[aug_trn_y == oc] = aug_signals
print('Augment Data Shape:', aug_trn_x.shape)
 trn_x = np.concatenate([orig_trn_x, aug_trn_x], axis=0)
 trn_y = np.concatenate([orig_trn_y, aug_trn_y], axis=0)
 plt.title('after augment'); plt.plot(trn_x[::1000]); plt.show()
plt.title('after augment label'); plt.plot(trn_y[::1000]); plt.show()
for oc in range(trn_y.max() +1):
signals = trn_x[trn_y == oc]
print(oc, signals.mean() , signals.var())
 res = pd.DataFrame()
res['signal'] = trn_x
res['open_channels'] = trn_y
all_dfs += [res]
df = pd.concat(all_dfs, axis=0 ).reset_index(drop=True)
 df = run_feat_engineering(df, batch_size=GROUP_BATCH_SIZE)
 tr = pd.concat([pd.get_dummies(df.open_channels), df[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
targets = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 signals = np.array(list(df.groupby('group' ).apply(lambda x: x[feats].values)))
 return signals, targets
def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[0])
 new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 orig_train = train.copy()
tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 f1_scores = []
for n_fold,(tr_idx, val_idx, trn_orig_ix, val_orig_idx)in enumerate(new_splits[0:], start=0):
print(f'Training fold {n_fold + 1} started')
 if n_fold not in [FOLD]:
continue
train_x, train_y = do_augment(orig_train, feats, trn_orig_ix, aug_time=AUG_CNT)
 valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier_Wavenet(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_scores += [f1_score_]
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 print('all fold f1 scores = ', f1_scores)
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 pd.to_pickle({
'train': oof_,
'test': preds_,
}, 'wavenet_aug_oof_{}_{}.pkl'.format(FOLD, AUG_CNT))
plt.plot(sample_submission['open_channels'].values[::1000]); plt.show()
def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = train_update_groups(train)
 test = test_update_groups(test)
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything() 
<load_from_csv>
input_layer = train_pre.shape[1]
output_layer = 2
num_epochs = 10001
learning_rate = 0.0001
train_size = 0.8
layers_dims = [input_layer, 256, 128, 64, output_layer]
warnings.filterwarnings('ignore')
 train_df = pd.read_csv("/kaggle/input/scaling-3/new_train.csv")
 test_df = pd.read_csv("/kaggle/input/scaling-3/new_test.csv")
 train_df['batch']=(( train_df.time-0.0001)//50 ).astype(int)
 test_df['batch']=(( test_df.time-0.0001)//50 ).astype(int)
 train_df['mini_batch']=(( train_df.time-0.0001)//10 ).astype(int)
 test_df['mini_batch']=(( test_df.time-0.0001)//10 ).astype(int)
 train_df['mini_mini_batch']=(( train_df.time-0.0001)//0.5 ).astype(int)
 test_df['mini_mini_batch']=(( test_df.time-0.0001)//0.5 ).astype(int)
 shifted = [4,9]
train_df.loc[train_df.batch.isin(shifted),'signal'] = train_df.loc[train_df.batch.isin(shifted),'signal']
shifted = [55,57]
test_df.loc[test_df.mini_batch.isin(shifted),'signal'] = test_df.loc[test_df.mini_batch.isin(shifted),'signal']
train_df = train_df.reset_index(drop=True)
 train_df.batch.value_counts()<groupby>
parameters, submission_name = model(X_train, Y_train, X_valid, Y_valid, layers_dims, num_epochs=num_epochs,
learning_rate=learning_rate, print_cost=False, plot_cost=True, l2_beta=0.1,
keep_prob=0.5, minibatch_size=0, return_best=True, print_accuracy=False,
plot_accuracy=True )
train_df.groupby(['batch','open_channels'] ).signal.agg(['mean','std'] )<import_modules>
final_prediction = predict(test_pre, parameters )
<groupby><EOS> 
submission = pd.DataFrame({"PassengerId":test.index.values})
 submission["Survived"] = np.argmax(final_prediction, 1)
 submission.to_csv("submission.csv", index=False )