Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train = pd.read_csv(f'train_folds.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv(f'{DATA_PATH}/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv(f'{BASE_PATH}/sample_submission.csv', dtype={'time': np.float32})
 y_train_proba = np.load(f"{RFC_DATA_PATH}/Y_train_proba.npy")
 y_test_proba = np.load(f"{RFC_DATA_PATH}/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = y_train_proba[:, i]
test[f"proba_{i}"] = y_test_proba[:, i]
train = create_rolling_features(train)
 test = create_rolling_features(test)
 train_mean = train.signal.mean()
train_std = train.signal.std()
train['signal'] =(train.signal - train_mean)/ train_std
test['signal'] =(test.signal - train_mean)/ train_std
print('Shape of train is ',train.shape)
 print('Shape of test is ',test.shape )<prepare_x_and_y>
from fastai import *
from fastai.tabular import *
from fastai.metrics import *
import numpy as np
import pandas as pd
batch_size = GROUP_BATCH_SIZE
train = create_features(train, batch_size)
 test = create_features(test, batch_size)
 cols_to_remove = ['time','signal','batch','batch_index','batch_slices','batch_slices2', 'group']
cols = [c for c in train.columns if c not in cols_to_remove]
cols_test = [c for c in test.columns if c not in cols_to_remove]
X = train[cols]
y = train['open_channels']
X_test = test[cols_test]<set_options>
path = Path(".. /input/titanic" )
del train
del test
gc.collect()<compute_test_metric>
df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv')
 df.head()
def f1_score_calc(y_true, y_pred):
return f1_score(y_true, y_pred, average="macro")
 def lgb_Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average="macro")
 return('KaggleMetric', score, True )<train_model>
class ZeroFare(TabularProc):
def apply_train(self, df:DataFrame):
df['Fare'] = df['Fare'].fillna(0)
 return df
class InspectName(TabularProc):
"Extracts valuable information from names, such as family name and title."
def apply_train(self, df:DataFrame):
df['FamilyName'] = df['Name'].apply(lambda s: s.split(",")[0])
 df['Title'] = df['Name'].apply(lambda s: s.split(", ")[1].split(".")[0] ).replace({'Mlle':'Miss', 'Mme':'Mrs', 'Ms':'Miss'})
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['Family'] = df['FamilyName'] + df['FamilySize'].apply(lambda i: "_" + str(i))
return df
class InspectTicket(TabularProc):
def apply_train(self, df:DataFrame):
df['TicketPrefix'] = df['Ticket'].apply(self.find_ticket_prefix)
 df['TicketClass'] = df['Pclass'].apply(lambda s: int(s))
df['TicketSequence'] = df['Ticket'].apply(self.find_ticket_sequence)
 return df
def find_ticket_prefix(self, s: str)-> str:
res = re.findall(r'[^\d]+', s)
 if len(res)> 0:
return res[0].strip()
else:
return None
def find_ticket_sequence(self, s: str)-> int:
res = re.findall(r'[\d]+$', s)
 if len(res)> 0:
return int(res[0])
 else:
return None
class InspectCabin(TabularProc):
def apply_train(self, df: DataFrame):
df['Deck'] = df['Cabin'].apply(self.extract_deck)
 df['CabinNumber'] = df['Cabin'].apply(self.extract_cabin_number)
 return df
def extract_deck(self, cabin: str)-> str:
if type(cabin)is float and math.isnan(cabin):
return None
res = re.findall(r'^(\w)', cabin)
 if len(res)> 0:
return res[0]
else:
return None
def extract_cabin_number(self, cabin: str)-> int:
if type(cabin)is float and math.isnan(cabin):
return None
res = re.findall(r'(\d+)', cabin)
 if len(res)> 0:
return int(res[0])
 else:
return None
class Motherhood(TabularProc):
def apply_train(self, df: DataFrame):
df['Mother'] = df[['Sex', 'Parch']].apply(self.determine_motherhood, axis=1)
 return df
def determine_motherhood(self, row)-> bool:
return row['Sex'] == "female" and row['Parch'] > 0
def train_model(X, X_test, params, model_type='lgb', eval_metric='f1score',
columns=None, plot_feature_importance=False, model=None,
verbose=50, early_stopping_rounds=200, n_estimators=2000):
metrics_dict = {'f1score': {'lgb_metric_name': lgb_Metric,}}
results = []
oof = np.zeros(len(X))
prediction = np.zeros(( len(X_test)))
 feature_importance = pd.DataFrame()
if True:
for fold in range(0,5):
result_dict = {}
X_train = X[X.kfold != fold].reset_index(drop=True)
 X_valid = X[X.kfold == fold].reset_index(drop=True)
 y_train = X_train.open_channels
y_valid = X_valid.open_channels
X_train = X_train.drop(['kfold', 'open_channels'], axis=1)
 X_valid = X_valid.drop(['kfold', 'open_channels'], axis=1)
 if model_type == 'lgb':
model = lgb.train(params, lgb.Dataset(X_train, y_train),
n_estimators, lgb.Dataset(X_valid, y_valid),
verbose_eval=verbose, early_stopping_rounds=early_stopping_rounds, feval=lgb_Metric)
 preds = model.predict(X_valid, num_iteration=model.best_iteration)
 y_pred_valid = np.round(np.clip(preds, 0, 10)).astype(int)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_set = xgb.DMatrix(X_train, y_train)
 val_set = xgb.DMatrix(X_valid, y_valid)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=verbose, early_stopping_rounds=early_stopping_rounds)
 preds = model.predict(xgb.DMatrix(X_valid))
y_pred_valid = np.round(np.clip(preds, 0, 10)).astype(int)
 y_pred = model.predict(xgb.DMatrix(X_test))
print(f'FINAL score fold {fold}: {f1_score_calc(y_valid, y_pred_valid)}')
 print('*'*100)
 result_dict['pred_valid'] = preds
result_dict['pred_test'] = y_pred
result_dict['model'] = model
results.append(result_dict)
 return results<train_model>
data.show_batch()
if TRAINING and MODEL_TYPE == 'lgb':
params = {'learning_rate': 0.1, 'max_depth': 7, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1}
result_dict_lgb = train_model(X=X, X_test=X_test, params=params, model_type=MODEL_TYPE, eval_metric='f1score', plot_feature_importance=False,
verbose=50, early_stopping_rounds=150, n_estimators=3000 )<train_model>
class Ranger(Optimizer):
def __init__(self, params, lr=1e-3, alpha=0.5, k=6, betas= (.9,0.999), eps=1e-8, weight_decay=0):
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
 if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
 if not lr > 0:
raise ValueError(f'Invalid Learning Rate: {lr}')
 if not eps > 0:
raise ValueError(f'Invalid eps: {eps}')
 defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super().__init__(params,defaults)
 for group in self.param_groups:
group["step_counter"] = 0
self.alpha = alpha
self.k = k
self.radam_buffer = [[None,None,None] for ind in range(10)]
self.slow_weights = [[p.clone().detach() for p in group['params']]
for group in self.param_groups]
for w in it.chain(*self.slow_weights):
w.requires_grad = False
def __setstate__(self, state):
print("set state called")
 super(Ranger, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.radam_buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma > 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma > 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 for group,slow_weights in zip(self.param_groups,self.slow_weights):
group['step_counter'] += 1
if group['step_counter'] % self.k != 0:
continue
for p,q in zip(group['params'],slow_weights):
if p.grad is None:
continue
q.data.add_(self.alpha,p.data - q.data)
 p.data.copy_(q.data)
 return loss
if TRAINING and MODEL_TYPE == 'xgb':
params_xgb = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse'}
result_dict_xgb = train_model(X=X, X_test=X_test, params=params_xgb, model_type=MODEL_TYPE, eval_metric='f1score', plot_feature_importance=False,
verbose=50, early_stopping_rounds=250 )<predict_on_test>
learn = tabular_learner(data,
layers=[50,30],
metrics=accuracy,
opt_func=Ranger,
emb_drop=0.1,
model_dir="/kaggle/working/models")
 learn.lr_find()
learn.recorder.plot(suggestion=True )
def get_prediction(test, model, model_type):
if model_type == 'xgb':
y_pred = model.predict(xgb.DMatrix(test))
else:
y_pred = model.predict(test, num_iteration=model.best_iteration)
 y_pred = np.round(np.clip(y_pred, 0, 10)).astype(int)
 return y_pred<load_from_csv>
learn.fit_one_cycle(3, 1e-02 )
if TRAINING:
sub = pd.read_csv(f'{BASE_PATH}/sample_submission.csv', dtype={'time': np.float32})
 if MODEL_TYPE == 'lgb':
preds = []
for fold in range(0,5):
preds.append(result_dict_lgb[fold][f'pred_test'])
 preds = np.average(preds, axis=0)
 y_pred = np.round(np.clip(preds, 0, 10)).astype(int)
 sub['open_channels'] = np.array(np.round(y_pred,0), np.int)
 elif MODEL_TYPE == 'xgb':
preds = []
for fold in range(0, 5):
preds.append(result_dict_xgb[fold][f'pred_test'])
 preds = np.average(preds, axis=0)
 y_pred = np.round(np.clip(preds, 0, 10)).astype(int)
 sub['open_channels'] = np.array(np.round(y_pred,0), np.int)
 sub.to_csv('submission.csv', index=False, float_format='%.4f')
 print(sub.head(10))
else:
if ENSEMBLE:
model_lgb = joblib.load(MODELS_PATH + 'lgb_0.sav')
 model_xgb = joblib.load(MODELS_PATH + 'xgb_0.sav')
 y_pred_lgb = get_prediction(X_test, model_lgb, 'lgb')
 y_pred_xgb = get_prediction(X_test, model_xgb, 'xgb')
 y_pred = 0.50 * y_pred_lgb + 0.50 * y_pred_xgb
else:
if MODEL_TYPE == 'lgb':
model_lgb = joblib.load(MODELS_PATH + 'lgb_0.sav')
 y_pred = get_prediction(X_test, model_lgb, MODEL_TYPE)
 elif MODEL_TYPE == 'xgb':
model_xgb = joblib.load(MODELS_PATH + 'xgb_1.sav')
 y_pred = get_prediction(X_test, model_xgb, MODEL_TYPE)
 sub = pd.read_csv(f'{BASE_PATH}/sample_submission.csv', dtype={'time': np.float32})
 sub['open_channels'] = np.array(np.round(y_pred,0), np.int)
 sub.to_csv('submission.csv', index=False, float_format='%.4f')
 print(sub.head(10))<load_from_csv>
learn.save('stage-1' )
train = pd.read_csv('.. /input/data-without-drift/train_clean.csv')
 test = pd.read_csv('.. /input/data-without-drift/test_clean.csv')
 train.head()<define_search_space>
learn.fit_one_cycle(10, 1e-02 )
batch_size = 500000
num_batches = 10
res = 1000
fs = 10000
nyq = 0.5 * fs
cutoff_freq_sweep = range(250,4750,50)
 lpf_cutoff = 600<statistical_test>
learn.save('stage-2' )
def butter_lowpass_filter(data, cutoff, fs, order):
normal_cutoff = cutoff / nyq
b, a = butter(order, normal_cutoff, btype='low', analog=False)
 y = filtfilt(b, a, data)
 return y
def butter_highpass_filter(data, cutoff, fs, order):
normal_cutoff = cutoff / nyq
b, a = butter(order, normal_cutoff, btype='high', analog=False)
 y = filtfilt(b, a, data)
 return y
def butter_bandpass_filter(data, cutoff_low, cuttoff_high, fs, order):
normal_cutoff_low = cutoff_low / nyq
normal_cutoff_high = cutoff_high / nyq
b, a = butter(order, [normal_cutoff_low,normal_cutoff_high], btype='band', analog=False)
 y = filtfilt(b, a, data)
 return y
def signaltonoise(a, axis=0, ddof=0):
a = np.asanyarray(a)
 m = a.mean(axis)
 sd = a.std(axis=axis, ddof=ddof)
 return np.where(sd == 0, 0, m/sd )<define_variables>
predictions, *_ = learn.get_preds(DatasetType.Test)
 labels = np.argmax(predictions, 1)
 labels
batch = 8
train['signal'][batch_size*(batch-1):batch_size*batch] = signal_lpf_batch_8
train['signal_undrifted'] = train['signal']
test['signal_undrifted'] = test['signal']<feature_engineering>
res_df = pd.DataFrame({'PassengerId': test_df['PassengerId'], 'Survived': labels})
 res_df.to_csv('titanic_submission.csv', index=False )