Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
pip install -U lightautoml
test_signal_shift_clean = []
test_signal_detrend = []
test_remove_shift = [True, True, False, False, True, False, True, True, True, False, True, False]
for data, use_fit, signal in zip(test_signal_shift, test_segm_is_shifted, test_signal):
if use_fit:
data_x = np.arange(len(data), dtype=float)* window_size + window_size/2
fit = np.flip(np.polyfit(data_x, data, 4))
data_x = np.arange(len(data)* window_size, dtype=float)
 data_2 = np.sum([ c * data_x ** i for i, c in enumerate(fit)], axis=0)
 else:
data_2 = np.zeros(len(data)* window_size, dtype=float)
 test_signal_detrend.append(signal + data_2 )<concatenate>
import os
import time
import re
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
import torch
from lightautoml.automl.presets.tabular_presets import TabularAutoML, TabularUtilizedAutoML
from lightautoml.dataset.roles import DatetimeRole
from lightautoml.tasks import Task
from lightautoml.utils.profiler import Profiler
test_signal = np.ndarray(0)
 for arr in test_signal_detrend:
test_signal = np.append(test_signal, arr )<split>
N_THREADS = 4
N_FOLDS = 5
RANDOM_STATE = 42
TEST_SIZE = 0.2
TIMEOUT = 600
train_time = train['time'][:].values
test_time = test['time'][:].values<drop_column>
np.random.seed(RANDOM_STATE)
 torch.set_num_threads(N_THREADS )
train = train.drop(['signal'], axis=1)
 test = test.drop(['signal'], axis=1 )<groupby>
%%time
train_data = pd.read_csv('.. /input/titanic/train.csv')
 train_data.head()
def data_filter(data, time, signal, sigm=6, batchsize=500000):
mean_std, cut_off, lower, upper, filtered_signal, filtered_time = [], [], [], [], [], []
for x, y in zip(data.groupby('batch')['signal'].agg('mean'), data.groupby('batch')['signal'].agg('std')) :
mean_std.append(( x, y))
for i in range(len(mean_std)) :
lower.append(mean_std[i][0] -(mean_std[i][1])*sigm)
 upper.append(mean_std[i][0] +(mean_std[i][1])*sigm)
 for i in range(len(data['batch'].value_counts())) :
for j in range(data['batch'].value_counts() [1]):
if(signal[j+i*batchsize] < upper[i])and(signal[j+i*batchsize] > lower[i]):
filtered_signal.append(signal[j+i*batchsize])
 filtered_time.append(time[j+i*batchsize])
 return filtered_time, filtered_signal<feature_engineering>
test_data = pd.read_csv('.. /input/titanic/test.csv')
 test_data.head()
train['signal'] = train_signal
test['signal'] = test_signal<split>
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission.head()
train_time, train_signal = data_filter(train, train_time, train_signal)
 test_time, test_signal = data_filter(test, test_time, test_signal, sigm=6, batchsize=500000 )<prepare_x_and_y>
def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
def create_extra_features(data):
data['Ticket_type'] = data['Ticket'].map(lambda x: x[0:3])
 data['Name_Words_Count'] = data['Name'].map(lambda x: len(x.split()))
data['Has_Cabin'] = data["Cabin"].map(lambda x: 1 - int(type(x)== float))
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['CategoricalFare'] = pd.qcut(data['Fare'], 5 ).astype(str)
 data['CategoricalAge'] = pd.cut(data['Age'], 5 ).astype(str)
 data['Title'] = data['Name'].apply(get_title ).replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data['Title'] = data['Title'].replace('Mlle', 'Miss')
 data['Title'] = data['Title'].replace('Ms', 'Miss')
 data['Title'] = data['Title'].replace('Mme', 'Mrs')
 data['Title'] = data['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5} ).fillna(0)
 return data
train_data = create_extra_features(train_data)
 test_data = create_extra_features(test_data )
def interpol(time, signal):
if len(time)> 2000000:
LENA =(0.0001, 500.0001, 0.0001)
 elif(len(time)> 500000)and(len(time)< 2000000):
LENA =(500.0001, 700.0001, 0.0001)
 else:
LENA =(350.0001, 400.0001, 0.0001)
 y = np.array(signal[:])
 x = np.array(time[:])
 f = interpolate.interp1d(x, y)
 new_time = np.arange(*LENA)
 new_signal = f(new_time)
 return new_time, new_signal<feature_engineering>
tr_data, te_data = train_test_split(train_data,
test_size=TEST_SIZE,
stratify=train_data['Survived'],
random_state=RANDOM_STATE)
 print('Data splitted.Parts sizes: tr_data = {}, te_data = {}'.format(tr_data.shape, te_data.shape))
train = train.drop(['signal', 'time'], axis=1)
 test = test.drop(['signal', 'time'], axis=1)
 train['time'], train_signal = interpol(train_time, train_signal)
 test['time'], test_signal = interpol(test_time, test_signal)
 train['signal'] = train_signal
test['signal'] = test_signal<import_modules>
%%time
def acc_score(y_true, y_pred, **kwargs):
return accuracy_score(y_true,(y_pred > 0.5 ).astype(int), **kwargs)
 task = Task('binary', metric = acc_score )
from scipy.signal import butter,filtfilt<init_hyperparams>
%%time
roles = {
'target': 'Survived',
'drop': ['PassengerId', 'Name','Ticket'],
}
T = 5.0
fs = 30.0
cutoff = 2
nyq = 0.5 * fs
order = 2
n = int(T * fs )<split>
%%time
automl = TabularAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
general_params = {'use_algos': [['linear_l2', 'lgb', 'lgb_tuned']]},
reader_params = {'n_jobs': N_THREADS})
 oof_pred = automl.fit_predict(tr_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred[:10], oof_pred.shape))
def butter_lowpass_filter(data, cutoff, fs, order):
normal_cutoff = cutoff / nyq
b, a = butter(order, normal_cutoff, btype='low', analog=False)
 y = filtfilt(b, a, data)
 return y<define_variables>
%%time
test_pred = automl.predict(te_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred[:10], test_pred.shape))
print('Check scores...')
 print('OOF score: {}'.format(acc_score(tr_data['Survived'].values, oof_pred.data[:, 0])))
 print('TEST score: {}'.format(acc_score(te_data['Survived'].values, test_pred.data[:, 0])) )
seventh_batch_time, seventh_batch_signal = [x/10000 for x in range(3500001, 4000001, 1)], list(train_signal[3500000:4000000] )<define_variables>
%%time
automl = TabularUtilizedAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
general_params = {'use_algos': [['linear_l2', 'lgb', 'lgb_tuned']]},
reader_params = {'n_jobs': N_THREADS})
 oof_pred = automl.fit_predict(tr_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred[:10], oof_pred.shape))
poper = 0
for i in range(1, 499999):
if(train_signal[i+3500000] > 2.2)or(train_signal[i+3500000] < -3.8):
seventh_batch_time.pop(i-poper)
 seventh_batch_signal.pop(i-poper)
 poper += 1<feature_engineering>
%%time
test_pred = automl.predict(te_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred[:10], test_pred.shape))
print('Check scores...')
 print('OOF score: {}'.format(acc_score(tr_data['Survived'].values, oof_pred.data[:, 0])))
 print('TEST score: {}'.format(acc_score(te_data['Survived'].values, test_pred.data[:, 0])) )
_, seventh_batch_signal = interpol(seventh_batch_time, seventh_batch_signal )<feature_engineering>
%%time
automl = TabularUtilizedAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
general_params = {'use_algos': [['linear_l2', 'lgb', 'lgb_tuned']]},
reader_params = {'n_jobs': N_THREADS})
 oof_pred = automl.fit_predict(train_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred[:10], oof_pred.shape))
train_signal[3500000:4000000] = seventh_batch_signal[:]<drop_column>
%%time
test_pred = automl.predict(test_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred[:10], test_pred.shape))
print('Check scores...')
 print('OOF score: {}'.format(acc_score(train_data['Survived'].values, oof_pred.data[:, 0])) )
<normalization><EOS> 
submission['Survived'] =(test_pred.data[:, 0] > 0.5 ).astype(int)
 submission.to_csv('automl_utilized_600.csv', index = False )