Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, auc, classification_report, confusion_matrix, f1_score, fbeta_score, precision_recall_curve, precision_score, recall_score, roc_auc_score, roc_curve
model = Classifier()
model<train_model>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
class IronDataset(Dataset):
def __init__(self, data, labels, training=True, transform=None, seq_len=5000, flip=0.5, noise_level=0, class_split=0.0):
self.data = data
self.labels = labels
self.transform = transform
self.training = training
self.flip = flip
self.noise_level = noise_level
self.class_split = class_split
self.seq_len = seq_len
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
data = self.data[idx]
labels = self.labels[idx]
return [data.astype(np.float32), labels.astype(int)]<split>
df_train['DataType'], df_test['DataType'] = 'training', 'testing'
df_test.insert(1, 'Survived', np.nan)
 df_data = pd.concat([df_train, df_test], ignore_index=True )
train, test, train_tr, new_splits = split()<install_modules>
col_convert = ['Survived', 'Pclass', 'SibSp', 'Parch']
df_data[col_convert] = df_data[col_convert].astype('object' )
pip install torchcontrib<install_modules>
df_data['Surname'] = df_data['Name'].str.extract(r'([A-Za-z]+),', expand=False )
!pip install pytorch_toolbelt<import_modules>
df_data['Title'] = df_data['Name'].str.extract(r'([A-Za-z]+)\.', expand=False)
 df_data['Title'] = df_data['Title'].replace(['Capt', 'Rev'], 'Crew')
 df_data['Title'] = df_data['Title'].replace('Ms', 'Miss')
 df_data['Title'] = df_data['Title'].replace(['Col', 'Countess', 'Don', 'Dona', 'Jonkheer', 'Lady', 'Major', 'Mlle', 'Mme', 'Sir'], 'Royal')
 df_data['Title'].value_counts()
from pytorch_toolbelt import losses as L<import_modules>
df_data['Age'] = df_data['Age'].fillna(df_data.groupby(['Title'], as_index=True)['Age'].transform('mean'))
from torchcontrib.optim import SWA
import torchcontrib<create_dataframe>
df_data['FamilySize'] = df_data['SibSp'] + df_data['Parch'] + 1
%%time
test_y = np.zeros([int(2000000/GROUP_BATCH_SIZE), GROUP_BATCH_SIZE, 1])
 test_dataset = IronDataset(test, test_y, flip=False)
 test_dataloader = DataLoader(test_dataset, NNBATCHSIZE, shuffle=False)
 test_preds_all = np.zeros(( 2000000, 11))
oof_score = []
for index,(train_index, val_index, _)in enumerate(new_splits[0:], start=0):
print("Fold : {}".format(index))
train_dataset = IronDataset(train[train_index], train_tr[train_index], seq_len=GROUP_BATCH_SIZE, flip=flip, noise_level=noise)
 train_dataloader = DataLoader(train_dataset, NNBATCHSIZE, shuffle=True,num_workers = 16)
 valid_dataset = IronDataset(train[val_index], train_tr[val_index], seq_len=GROUP_BATCH_SIZE, flip=False)
 valid_dataloader = DataLoader(valid_dataset, NNBATCHSIZE, shuffle=False)
 it = 0
model = Classifier()
model = model.cuda()
early_stopping = EarlyStopping(patience=40, is_maximize=True,
checkpoint_path=os.path.join(outdir, "gru_clean_checkpoint_fold_{}_iter_{}.pt".format(index,
it)))
 weight = None
criterion = L.FocalLoss()
avg_train_losses, avg_valid_losses = [], []
for epoch in range(EPOCHS):
if epoch < 1:
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 elif epoch < 7:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/3)
 elif epoch < 50:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/5)
 elif epoch < 60:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/7)
 elif epoch < 70:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/9)
 elif epoch < 80:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/11)
 elif epoch < 90:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/13)
 else:
optimizer = torch.optim.Adam(model.parameters() , lr=LR/100)
 optimizer = torchcontrib.optim.SWA(optimizer, swa_start=2, swa_freq=2, swa_lr=0.00011)
 train_losses, valid_losses = [], []
tr_loss_cls_item, val_loss_cls_item = [], []
model.train()
train_preds, train_true = torch.Tensor([] ).cuda() , torch.LongTensor([] ).cuda()
print('**********************************')
 print("Folder : {} Epoch : {}".format(index, epoch))
print("Curr learning_rate: {:0.9f}".format(optimizer.param_groups[0]['lr']))
for x, y in tqdm(train_dataloader):
x = x.cuda()
y = y.cuda()
optimizer.zero_grad()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 y_ = y.view(-1)
 loss = criterion(predictions_, y_)
 loss.backward()
optimizer.step()
train_losses.append(loss.item())
 train_true = torch.cat([train_true, y_], 0)
 train_preds = torch.cat([train_preds, predictions_], 0)
 if epoch >= 90:
optimizer.update_swa()
optimizer.swap_swa_sgd()
val_preds, val_true = torch.Tensor([] ).cuda() , torch.LongTensor([] ).cuda()
print('EVALUATION')
 with torch.no_grad() :
for x, y in tqdm(valid_dataloader):
x = x.cuda()
y = y.cuda()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 y_ = y.view(-1)
 loss = criterion(predictions_, y_)
 valid_losses.append(loss.item())
 val_true = torch.cat([val_true, y_], 0)
 val_preds = torch.cat([val_preds, predictions_], 0)
 train_loss = np.average(train_losses)
 valid_loss = np.average(valid_losses)
 avg_train_losses.append(train_loss)
 avg_valid_losses.append(valid_loss)
 print("train_loss: {:0.6f}, valid_loss: {:0.6f}".format(train_loss, valid_loss))
train_score = f1_score(train_true.cpu().detach().numpy() , train_preds.cpu().detach().numpy().argmax(1),
labels=list(range(11)) , average='macro')
 val_score = f1_score(val_true.cpu().detach().numpy() , val_preds.cpu().detach().numpy().argmax(1),
labels=list(range(11)) , average='macro')
 print("train_f1: {:0.6f}, valid_f1: {:0.6f}".format(train_score, val_score))
res = early_stopping(val_score, model)
 if res == 2:
print("Early Stopping")
 print('folder %d global best val max f1 model score %f' %(index, early_stopping.best_score))
break
elif res == 1:
print('save folder %d global val max f1 model score %f' %(index, val_score))
print('Folder {} finally best global max f1 score is {}'.format(index, early_stopping.best_score))
oof_score.append(round(early_stopping.best_score, 6))
model.eval()
pred_list = []
with torch.no_grad() :
for x, y in tqdm(test_dataloader):
x = x.cuda()
y = y.cuda()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 pred_list.append(F.softmax(predictions_, dim=1 ).cpu().numpy())
 test_preds = np.vstack(pred_list)
 test_preds_all += test_preds 
<save_to_csv>
df_data['TicketString'] = df_data['Ticket'].apply(lambda x: ''.join(re.findall(r'[a-zA-Z]+', x)))
 df_data['TicketString'] = df_data['TicketString'].replace(['CASOTON', 'SOTONO', 'STONO', 'STONOQ'], 'SOTONOQ')
 df_data['TicketString'] = df_data['TicketString'].replace(['SC', 'SCParis'], 'SCPARIS')
 df_data['TicketString'] = df_data['TicketString'].replace('FCC', 'FC')
 df_data['TicketString'] = df_data['TicketString'].replace(df_data['TicketString'].value_counts() [df_data['TicketString'].value_counts() < 10].index.tolist() , 'OTHER')
 df_data['TicketString'].value_counts()
print('all folder score is:%s'%str(oof_score))
print('OOF mean score is: %f'%(sum(oof_score)/len(oof_score)))
 print('Generate submission.............')
 submission_csv_path = '/kaggle/input/liverpool-ion-switching/sample_submission.csv'
ss = pd.read_csv(submission_csv_path, dtype={'time': str})
 test_preds_all = test_preds_all / np.sum(test_preds_all, axis=1)[:, None]
test_pred_frame = pd.DataFrame({'time': ss['time'].astype(str),
'open_channels': np.argmax(test_preds_all, axis=1)})
 test_pred_frame.to_csv("./gru_preds.csv", index=False)
 print('over' )<set_options>
df_data['HasTicketString'] = df_data['TicketString'].apply(lambda x: 1 if x else 0 ).astype('object' )
!pip install tensorflow_addons
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
df_data['Fare'] = df_data['Fare'].fillna(df_data.groupby(['Pclass'], as_index=True)['Fare'].transform('mean'))
train = pd.read_csv('.. /input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('.. /input/liverpool-ion-switching/test.csv')
 ss = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv' )<define_variables>
df_data['Cabin'] = df_data['Cabin'].fillna(0 )
def make_batches(df, dataset="train"):
batches = []
batch_size = [500000, 100000]
if dataset == "train":
for idx in range(10):
batches.append(df[idx * batch_size[0]:(idx + 1)* batch_size[0]])
 else:
for idx in range(10):
batches.append(df[idx * batch_size[1]:(idx + 1)* batch_size[1]])
 for idx in range(2):
base = 10 * batch_size[1]
batches.append(df[base + idx * batch_size[0]: base +(idx + 1)* batch_size[0]])
 return batches
train_batches = make_batches(train, "train")
 test_batches = make_batches(test, "test" )<normalization>
df_data['CabinString'] = df_data['Cabin'].str.extract(r'([A-Za-z]+)', expand=False )
linear_train_idx = [1]
linear_test_idx = [0, 1, 4, 6, 7, 8]
def linear_drift(x, x0):
return 0.3 *(x - x0)
 def remove_linear_drift(data, dataset="train"):
if dataset == "train":
data[1].loc[data[1].index[0:100000], 'signal'] = data[1].signal[0:100000].values - linear_drift(data[1].time[0:100000].values, data[1].time[0:1].values)
 else:
for idx in linear_test_idx:
data[idx].loc[data[idx].index[0:100000], 'signal'] = data[idx].signal[0:100000].values - linear_drift(data[idx].time[0:100000].values, data[idx].time[0:1].values)
 return data
train_drift_removed = remove_linear_drift(train_batches, "train")
 test_drift_removed = remove_linear_drift(test_batches, "test" )<create_dataframe>
df_data['HasCabin'] = df_data['CabinString'].apply(lambda x: 0 if pd.isnull(x)else 1 ).astype('object' )
df_train = train_drift_removed.copy()
df_test = test_drift_removed.copy()<train_model>
df_data['Embarked'] = df_data['Embarked'].fillna(df_data['Embarked'].value_counts().idxmax() )
def my_sin(x, A, ph, d):
frequency = 0.01
omega = 2 * np.pi * frequency
return A * np.sin(omega * x + ph)+ d
def parabolic_drift_fit(data):
x = data['time']
y = data['signal']
frequency = 0.01
omega = 2 * np.pi * frequency
M = np.array([[np.sin(omega * t), np.cos(omega * t), 1] for t in x])
 y = np.array(y ).reshape(len(y), 1)
(theta, _, _, _)= np.linalg.lstsq(M, y)
 A = np.sqrt(theta[0,0]**2 + theta[1,0]**2)
 ph = math.atan2(theta[1,0], theta[0,0])
 d = theta[2,0]
popt = [A, ph, d]
print(popt)
 return popt
parabola_params = []
for idx in parabola_train_idx:
parabola_params.append(parabolic_drift_fit(df_train[idx]))
parabola_params.append(parabolic_drift_fit(df_test[parabola_test_idx[0]]))
plt.figure(figsize=(30, 4))
for n, idx in enumerate(parabola_train_idx):
plt.subplot("15" + str(n + 1))
plt.title("Train " + str(idx))
plt.ylabel("Signal", fontsize=8)
 plt.plot(df_train[idx]['time'], df_train[idx]['signal'], linewidth=.1)
 plt.plot(df_train[idx]['time'], my_sin(df_train[idx]['time'], *parabola_params[n]), 'y')
 plt.grid()
plt.ylim([np.min(df_train[idx]['signal']), np.min(df_train[idx]['signal'])+ 18])
 plt.subplot("155")
 plt.title("Test 10")
 plt.ylabel("Signal", fontsize=8)
 plt.ylim([np.min(df_test[10]['signal']), np.min(df_test[10]['signal'])+ 18])
 plt.plot(df_test[10]['time'], df_test[10]['signal'], linewidth=.1)
 plt.plot(df_test[10]['time'], my_sin(df_test[10]['time'], *parabola_params[-1]), 'y')
 plt.grid()<train_model>
df_ticket = pd.get_dummies(df_data[df_data['FamilySize'] > 1], columns=['Pclass', 'Sex', 'Embarked', 'DataType', 'Title', 'CabinString', 'HasCabin'], drop_first=False)
 df_ticket['Survived'] = df_ticket['Survived'].astype(float)
 df_ticket = df_ticket.groupby(['Ticket'], as_index=False ).agg({
'Survived': 'mean',
'Pclass_1': sum, 'Pclass_2': sum, 'Pclass_3': sum,
'Sex_male': sum, 'Sex_female': sum,
'Embarked_C': sum, 'Embarked_Q': sum, 'Embarked_S': sum,
'DataType_training': sum, 'DataType_testing': sum,
'Title_Crew': sum, 'Title_Dr': sum, 'Title_Master': sum, 'Title_Miss': sum, 'Title_Mr': sum, 'Title_Mrs': sum, 'Title_Royal': sum,
'CabinString_A': sum, 'CabinString_B': sum, 'CabinString_C': sum, 'CabinString_D': sum, 'CabinString_E': sum, 'CabinString_F': sum, 'CabinString_G': sum,
'HasCabin_0': sum, 'HasCabin_1': sum
} )
def my_sin(x, A, ph, d):
frequency = 0.01
omega = 2 * np.pi * frequency
return A * np.sin(omega * x + ph)+ d
def parabolic_drift_fit(data):
x = data['time']
y = data['signal']
frequency = 0.01
omega = 2 * np.pi * frequency
M = np.array([[np.sin(omega * t), np.cos(omega * t), 1] for t in x])
 y = np.array(y ).reshape(len(y), 1)
(theta, _, _, _)= np.linalg.lstsq(M, y)
 A = np.sqrt(theta[0,0]**2 + theta[1,0]**2)
 ph = math.atan2(theta[1,0], theta[0,0])
 d = theta[2,0]
popt = [A, ph, d]
print(popt)
 return popt
parabola_params = []
for idx in parabola_train_idx:
parabola_params.append(parabolic_drift_fit(df_train[idx]))
parabola_params.append(parabolic_drift_fit(df_test[parabola_test_idx[0]]))
plt.figure(figsize=(30, 4))
for n, idx in enumerate(parabola_train_idx):
plt.subplot("15" + str(n + 1))
plt.title("Train " + str(idx))
plt.ylabel("Signal", fontsize=8)
 plt.plot(df_train[idx]['time'], df_train[idx]['signal'], linewidth=.1)
 plt.plot(df_train[idx]['time'], my_sin(df_train[idx]['time'], *parabola_params[n]), 'y')
 plt.grid()
plt.ylim([np.min(df_train[idx]['signal']), np.min(df_train[idx]['signal'])+ 18])
 plt.subplot("155")
 plt.title("Test 10")
 plt.ylabel("Signal", fontsize=8)
 plt.ylim([np.min(df_test[10]['signal']), np.min(df_test[10]['signal'])+ 18])
 plt.plot(df_test[10]['time'], df_test[10]['signal'], linewidth=.1)
 plt.plot(df_test[10]['time'], my_sin(df_test[10]['time'], *parabola_params[-1]), 'y')
 plt.grid()<normalization>
col_convert = df_ticket.columns.drop('Ticket' ).tolist()
df_ticket[col_convert] = df_ticket[col_convert].astype('object' )
parabola_train_idx = [6, 7, 8, 9]
parabola_test_idx = [10]
def parabolic_drift(x, t=0):
f = 0.01
omega = 2 * np.pi * f
return 5 * np.sin(omega * x + t * np.pi)
 def remove_parabolic_drift(data, dataset="train"):
if dataset == "train":
for idx in parabola_train_idx:
data[idx].loc[data[idx].index[0:500000], 'signal'] = data[idx].signal[0:500000].values - parabolic_drift(data[idx].time[0:500000].values,(idx % 2))
else:
data[10].loc[data[10].index[0:500000], 'signal'] = data[10].signal[0:500000].values - parabolic_drift(data[10].time[0:500000].values)
 return data
df_train = remove_parabolic_drift(train_drift_removed, "train")
 df_test = remove_parabolic_drift(test_drift_removed, "test" )<save_to_csv>
col_convert = ['Survived']
df_ticket[col_convert] = df_ticket[col_convert].astype(float )
df_train_clean = df_train[0]
df_test_clean = df_test[0]
for df in df_train[1:]:
df_train_clean = pd.concat([df_train_clean, df], ignore_index=True)
 for df in df_test[1:]:
df_test_clean = pd.concat([df_test_clean, df], ignore_index=True)
 df_train_clean.to_csv("train_wo_drift.csv", index=False, float_format="%.4f")
 df_test_clean.to_csv("test_wo_drift.csv", index=False, float_format="%.4f" )<load_from_csv>
df_ticket['Together'] = df_ticket['Survived'].apply(lambda x: 1 if x == 0 or x == 1 else 0 ).astype('object' )
def read_data() :
train = pd.read_csv('/kaggle/working/train_wo_drift.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/working/test_wo_drift.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub<load_from_disk>
df_data = pd.merge(df_data, df_ticket[['Ticket', 'Sex_male', 'Sex_female', 'Title_Crew', 'Title_Dr', 'Title_Master', 'Title_Miss', 'Title_Mr', 'Title_Mrs', 'Title_Royal']], how='left', left_on='Ticket', right_on='Ticket' ).rename(columns={
'Sex_male': 'WithSexMale', 'Sex_female': 'WithSexFemale',
'Title_Crew': 'WithTitleCrew', 'Title_Dr': 'WithTitleDr', 'Title_Master': 'WithTitleMaster', 'Title_Miss': 'WithTitleMiss', 'Title_Mr': 'WithTitleMr', 'Title_Mrs': 'WithTitleMrs', 'Title_Royal': 'WithTitleRoyal'
})
 col_fillnas = ['WithSexMale', 'WithSexFemale', 'WithTitleCrew', 'WithTitleDr', 'WithTitleMaster', 'WithTitleMiss', 'WithTitleMr', 'WithTitleMrs', 'WithTitleRoyal']
for col_fillna in col_fillnas: df_data[col_fillna] = df_data[col_fillna].fillna(0 )
read_data()<define_variables>
df_temp = df_data.copy(deep=True)
 df_temp['Survived'] = df_temp['Survived'].astype(float)
 df_ticket_self = df_temp.groupby(['Ticket'], as_index=True)
 count = df_ticket_self['Survived'].transform('count')
 mean = df_ticket_self['Survived'].transform('mean')
 df_data['SurvivedPeer'] =(mean * count - df_data['Survived'].astype(float)) /(count - 1)
 df_data['SurvivedPeer'] = df_data['SurvivedPeer'].astype(float ).fillna(-1 )
EPOCHS = 100
NNBATCHSIZE = 32
GROUP_BATCH_SIZE = 4000
SEED = 123
LR = 0.001
SPLITS = 4
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<categorify>
df_temp = df_data.copy(deep=True)
 df_temp['Survived'] = df_temp['Survived'].astype(float)
 col_revises = ['Crew', 'Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Royal']
for col_revise in col_revises:
df_temp['Survived' + col_revise] = df_temp['Survived']
df_temp.loc[df_temp['Title'] != col_revise, 'Survived' + col_revise] = np.nan
df_ticket_title = df_temp.groupby(['Ticket'], as_index=True)
 for col_revise in col_revises:
count = df_ticket_title['Survived' + col_revise].transform('count')
 mean = df_ticket_title['Survived' + col_revise].transform('mean')
 df_data['SurvivedPeer' + col_revise] =(mean * count - df_temp['Survived' + col_revise].astype(float)) /(count - 1)
 df_data['SurvivedPeer' + col_revise] = df_data['SurvivedPeer' + col_revise].astype(float ).fillna(-1 )
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = wave_block(inp, 32, 3, 12)
 x = wave_block(x, 64, 3, 8)
 x = wave_block(x, 64, 3, 4)
 x = wave_block(x, 128, 3, 1)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=4)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything()<set_options>
df_data['Survived'] = df_data['Survived'].fillna(-1 )
warnings.filterwarnings("ignore")
 n_leaves = 32
depth = 6
n_shift = 2<load_from_csv>
df_data = pd.get_dummies(df_data, columns=['Pclass', 'Sex', 'Embarked', 'DataType', 'Title', 'TicketString', 'HasTicketString', 'CabinString', 'HasCabin'], drop_first=True )
train = pd.read_csv('.. /input/data-without-drift-with-kalman-filter/train.csv')
 test = pd.read_csv('.. /input/data-without-drift-with-kalman-filter/test.csv')
 submission = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv' )<categorify>
col_convert = ['Survived', 'SibSp', 'Parch', 'FamilySize']
df_data[col_convert] = df_data[col_convert].astype(int )
def shift(df, n):
for i in range(1,n+1):
df['signal-' + str(i)] = [0]*i + list(df['signal'].values[:-i])
 df['signal+' + str(i)] = list(df['signal'].values[i:])+ [0]*i
return df<feature_engineering>
x = df_data[df_data['DataType_training'] == 1].drop(['PassengerId', 'Survived', 'Name', 'Ticket', 'Cabin', 'Surname', 'FamilySize', 'DataType_training', 'SurvivedPeerCrew', 'SurvivedPeerDr', 'SurvivedPeerMaster', 'SurvivedPeerMiss', 'SurvivedPeerMr', 'SurvivedPeerMrs', 'SurvivedPeerRoyal'], axis=1)
 y = df_data.loc[df_data['DataType_training'] == 1, 'Survived']
t = train[:500000]
outlier = list(t[(t.signal>-1)&(t.open_channels==0)].index)
 train.at[outlier, 'signal'] = -1.6
t = train[500000:1000000]
outlier = list(t[(t.signal>0)&(t.open_channels==0)].index)
 train.at[outlier, 'signal'] = -1.6
del t<concatenate>
forestclf = RandomForestClassifier(n_estimators=100, random_state=58 ).fit(x, y)
 feat = pd.DataFrame(data=forestclf.feature_importances_, index=x.columns, columns=['FeatureImportances'] ).sort_values(['FeatureImportances'], ascending=False )
dic_model = {}
d = {}
for i in range(10):
d['batch{0}'.format(i)] = shift(train[500000*i:500000*(i+1)], n_shift ).drop('time',axis=1)
 df1 = pd.concat([d['batch0'],d['batch1'],d['batch2'],d['batch3'],d['batch5'],d['batch6'],d['batch8']], axis=0 ).sample(frac=1)
 df2 = pd.concat([d['batch4'], d['batch9']], axis=0 )<drop_column>
model_feat = feat[feat['FeatureImportances'] > 0.005].index
df1 = reduce_mem_usage(normal_distribution_feature(df1, 1))
df2 = reduce_mem_usage(normal_distribution_feature(df2, 2))<train_model>
x = df_data.loc[df_data['DataType_training'] == 1, model_feat]
y = df_data.loc[df_data['DataType_training'] == 1, 'Survived']
def fit_lgb(X, y, upper_bound):
lomodel = []
kf = KFold(n_splits=5, random_state=1, shuffle=False)
 params = {'objective': 'huber',
'num_leaves': n_leaves,
'max_depth':depth,
"metric": 'rmse',
'n_jobs': -1,
'random_state': 1
}
lopred=[]
for train_index, test_index in kf.split(X):
X_train, X_valid = X.iloc[train_index], X.iloc[test_index]
y_train, y_valid = y.iloc[train_index], y.iloc[test_index]
d_train = lgb.Dataset(X_train, label=y_train)
 d_valid = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,
train_set=d_train,
num_boost_round=1000,
valid_sets=d_valid,
verbose_eval='None',
early_stopping_rounds=30,
learning_rates=lambda iter:max(0.1**(iter//10),0.001), 
)
 lomodel.append(model)
 pred = model.predict(X_valid, num_iteration=model.best_iteration)
 pred = np.round(np.clip(pred,0,upper_bound)).astype(int)
 lopred.append(pred)
 print(f1_score(y_valid, pred, average=None))
print('
')
 return lomodel,lopred
dic_model['lomodel1'],lopred1 = fit_lgb(df1.drop(['open_channels'], axis = 1), df1.open_channels, 5)
 dic_model['lomodel2'],lopred2 = fit_lgb(df2.drop(['open_channels'], axis = 1), df2.open_channels, 10 )<compute_test_metric>
x_train, x_validate, y_train, y_validate = train_test_split(x, y, test_size=0.25, random_state=58 )
lopred = lopred1+lopred2
pred = np.array([])
 for i in lopred:
pred = np.append(pred, i)
 print('f1 score for each group:',f1_score(np.concatenate(( df1.open_channels,df2.open_channels)) , pred, average=None),
'macro f1 score:',f1_score(np.concatenate(( df1.open_channels,df2.open_channels)) , pred, average='macro'))<save_to_csv>
model_logreg = LogisticRegression(solver='lbfgs', max_iter=1024)
 model_logreg.fit(x_train, y_train)
 model_logreg_ypredict = model_logreg.predict(x_validate)
 model_logreg_f1score = f1_score(y_validate, model_logreg_ypredict)
 model_logreg_accuracyscore = accuracy_score(y_validate, model_logreg_ypredict)
 model_logreg_cvscores = cross_val_score(model_logreg, x, y, cv=20, scoring='accuracy')
 print('logistic regression
f1 score: %0.4f, accuracy score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_logreg_f1score, model_logreg_accuracyscore, model_logreg_cvscores.mean() , 2 * model_logreg_cvscores.std()))
submission['open_channels'] = np.array(np.round(sub,0), np.int)
 submission.to_csv('submission.csv', index=False, float_format='%.4f' )<set_options>
model_treeclf = DecisionTreeClassifier(splitter='best', min_samples_split=5)
 model_treeclf.fit(x_train, y_train)
 model_treeclf_ypredict = model_treeclf.predict(x_validate)
 model_treeclf_f1score = f1_score(y_validate, model_treeclf_ypredict)
 model_treeclf_accuracyscore = accuracy_score(y_validate, model_treeclf_ypredict)
 model_treeclf_cvscores = cross_val_score(model_treeclf, x, y, cv=20, scoring='accuracy')
 print('decision tree classifier
f1 score: %0.4f, accuracy score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_treeclf_f1score, model_treeclf_accuracyscore, model_treeclf_cvscores.mean() , 2 * model_treeclf_cvscores.std()))
!pip install tensorflow_addons
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>
model_forestclf = RandomForestClassifier(n_estimators=100, min_samples_split=5, random_state=58)
 model_forestclf.fit(x_train, y_train)
 model_forestclf_ypredict = model_forestclf.predict(x_validate)
 model_forestclf_f1score = f1_score(y_validate, model_forestclf_ypredict)
 model_forestclf_accuracyscore = accuracy_score(y_validate, model_forestclf_ypredict)
 model_forestclf_cvscores = cross_val_score(model_forestclf, x, y, cv=20, scoring='accuracy')
 print('random forest classifier
f1 score: %0.4f, accuracy score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_forestclf_f1score, model_forestclf_accuracyscore, model_forestclf_cvscores.mean() , 2 * model_forestclf_cvscores.std()))
EPOCHS = 110
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.001
SPLITS = 5
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<load_from_csv>
params = {'n_estimators': [100],
'max_depth': [10, 20, None],
'min_samples_split': [3, 5, 7, 9],
'random_state': [58],
}
model_forestclf_cv = GridSearchCV(model_forestclf, params, cv=5)
 model_forestclf_cv.fit(x_train, y_train)
 model_forestclf_cv_ypredict = model_forestclf_cv.predict(x_validate)
 model_forestclf_cv_f1score = f1_score(y_validate, model_forestclf_cv_ypredict)
 model_forestclf_cv_accuracyscore = accuracy_score(y_validate, model_forestclf_cv_ypredict)
 model_forestclf_cv_cvscores = cross_val_score(model_forestclf_cv, x, y, cv=20, scoring='accuracy')
 print('random forest classifier grid search
f1 score: %0.4f, accuracy score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_forestclf_cv_f1score, model_forestclf_cv_accuracyscore, model_forestclf_cv_cvscores.mean() , 2 * model_forestclf_cv_cvscores.std()))
print(' best parameters: %s' %model_forestclf_cv.best_params_ )
def read_data() :
train = pd.read_csv('/kaggle/input/clean-kalman/train_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/clean-kalman/test_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = wave_block(inp, 16, 3, 12)
 x = wave_block(x, 32, 3, 8)
 x = wave_block(x, 64, 3, 4)
 x = wave_block(x, 128, 3, 1)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything()<set_options>
final_model = model_forestclf
x_test = df_data.loc[df_data['DataType_training'] == 0, model_feat]
y_test = pd.DataFrame(final_model.predict(x_test), columns=['Survived'], index=df_data.loc[df_data['DataType_training'] == 0, 'PassengerId'] )
<init_hyperparams><EOS> 
out = pd.DataFrame({'PassengerId': y_test.index, 'Survived': y_test['Survived']})
 out.to_csv('submission.csv', index=False )