Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
warnings.filterwarnings("ignore")
 sns.set(style='white', context='notebook', palette='deep' )
class Bi_RNN(nn.Module):
def __init__(self, input_dim, hidden_dim, batch_size, output_dim=11, num_layers=2, rnn_type='LSTM'):
super(Bi_RNN, self ).__init__()
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.batch_size = batch_size
self.num_layers = num_layers
self.init_linear = nn.Linear(self.input_dim, self.input_dim)
 self.lstm = eval('nn.' + rnn_type )(self.input_dim, self.hidden_dim, self.num_layers, batch_first=True, bidirectional=True)
 self.linear = nn.Linear(self.hidden_dim * 2, output_dim)
 def init_hidden(self):
return(torch.zeros(self.num_layers, self.batch_size, self.hidden_dim),
torch.zeros(self.num_layers, self.batch_size, self.hidden_dim))
def forward(self, input):
linear_input = self.init_linear(input)
 lstm_out, self.hidden = self.lstm(linear_input)
 y_pred = self.linear(lstm_out)
 return y_pred<prepare_x_and_y>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 combine = pd.concat([train.drop('Survived',1),test] )
class ION_Dataset_Sequential(Dataset):
def __init__(self, input, output):
self.input = input
self.output = output
def __len__(self):
return len(self.input)
 def __getitem__(self, idx):
x = self.input[idx]
y = self.output[idx]
x = torch.tensor(x, dtype=torch.float)
 y = torch.tensor(y, dtype=torch.float)
 return x, y
class ION_Dataset_Sequential_test(Dataset):
def __init__(self, input):
self.input = input
def __len__(self):
return len(self.input)
 def __getitem__(self, idx):
x = self.input[idx]
x = torch.tensor(x, dtype=torch.float)
 return x<load_from_csv>
surv = train[train['Survived']==1]
nosurv = train[train['Survived']==0]
surv_col = "blue"
nosurv_col = "red"
print("Survived: %i(%.1f percent), Not Survived: %i(%.1f percent), Total: %i"\
%(len(surv), 1.*len(surv)/len(train)*100.0,\
len(nosurv), 1.*len(nosurv)/len(train)*100.0, len(train)) )
train_df = pd.read_csv('/kaggle/input/data-no-drift/train_detrend.csv')
 test_df = pd.read_csv('/kaggle/input/data-no-drift/test_detrend.csv')
 X = train_df['signal'].values.reshape(-1, num_time_steps, 1)
 y = pd.get_dummies(train_df['open_channels'] ).values.reshape(-1, num_time_steps, output_dim)
 test_input = test_df["signal"].values.reshape(-1, num_time_steps, 1)
 train_input_mean = X.mean()
train_input_sigma = X.std()
test_input =(test_input-train_input_mean)/train_input_sigma
test_preds = np.zeros(( int(test_input.shape[0] * test_input.shape[1])))
 test = ION_Dataset_Sequential_test(test_input)
 test_loader = DataLoader(test, batch_size=batch_size, shuffle=False )<split>
print("There are %i unique ticket numbers among the %i tickets." \
%(train['Ticket'].nunique() ,train['Ticket'].count()))
kfold = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 local_val_score = 0
models = {}
k=0
for tr_idx, val_idx in kfold.split(X, y):
test_p = np.zeros(( int(test_input.shape[0] * test_input.shape[1])))
 print('starting fold', k)
 k += 1
print(6*'
train_input = X[tr_idx]
print(train_input.shape)
 train_target = y[tr_idx]
val_input = X[val_idx]
val_target = y[val_idx]
train_input_mean = train_input.mean()
train_input_sigma = train_input.std()
val_input =(val_input-train_input_mean)/train_input_sigma
train_input =(train_input-train_input_mean)/train_input_sigma
print(6*'
train = ION_Dataset_Sequential(train_input, train_target)
 valid = ION_Dataset_Sequential(val_input, val_target)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 train_preds = np.zeros(( int(train_input.shape[0] * train_input.shape[1])))
 val_preds = np.zeros(( int(val_input.shape[0] * val_input.shape[1])))
 best_val_preds = np.zeros(( int(val_input.shape[0] * val_input.shape[1])))
 train_targets = np.zeros(( int(train_input.shape[0] * train_input.shape[1])))
 avg_losses_f = []
avg_val_losses_f = []
loss_fn = torch.nn.BCEWithLogitsLoss()
model = Bi_RNN(lstm_input_size, hidden_state_size, batch_size=batch_size, output_dim=output_dim, num_layers=num_sequence_layers, rnn_type=rnn_type)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=1e-5)
 scheduler = ReduceLROnPlateau(optimizer, 'min', patience=150, factor=0.1, min_lr=1e-8)
 temp_val_loss = 9999999999
reached_val_score = 0
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch)in enumerate(train_loader):
x_batch = x_batch.view(-1, num_time_steps, lstm_input_size)
 y_batch = y_batch.view(-1, num_time_steps, output_dim)
 optimizer.zero_grad()
y_pred = model(x_batch.cuda())
 loss = loss_fn(y_pred.cpu() , y_batch)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 pred = F.softmax(y_pred, 2 ).detach().cpu().numpy().argmax(axis=-1)
 train_preds[i * batch_size * train_input.shape[1]:(i + 1)* batch_size * train_input.shape[1]] = pred.reshape(( -1))
train_targets[i * batch_size * train_input.shape[1]:(i + 1)* batch_size * train_input.shape[1]] = y_batch.detach().cpu().numpy().argmax(axis=2 ).reshape(( -1))
del y_pred, loss, x_batch, y_batch, pred
model.eval()
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
x_batch = x_batch.view(-1, num_time_steps, lstm_input_size)
 y_batch = y_batch.view(-1, num_time_steps, output_dim)
 y_pred = model(x_batch.cuda() ).detach()
avg_val_loss += loss_fn(y_pred.cpu() , y_batch ).item() / len(valid_loader)
 pred = F.softmax(y_pred, 2 ).detach().cpu().numpy().argmax(axis=-1)
 val_preds[i * batch_size * val_input.shape[1]:(i + 1)* batch_size * val_input.shape[1]] = pred.reshape(( -1))
del y_pred, x_batch, y_batch, pred
if avg_val_loss < temp_val_loss:
temp_val_loss = avg_val_loss
train_score = f1_score(train_targets, train_preds, average='macro')
 val_score = f1_score(val_target.argmax(axis=2 ).reshape(( -1)) , val_preds, average='macro')
 elapsed_time = time.time() - start_time
scheduler.step(avg_val_loss)
 if epoch%10 == 0:
print('Epoch {}/{} \t loss={:.4f} \t train_f1={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} \t time={:.2f}s'.format(epoch + 1, n_epochs, avg_loss, train_score, avg_val_loss, val_score, elapsed_time))
if val_score > reached_val_score:
reached_val_score = val_score
best_model = copy.deepcopy(model.state_dict())
 best_val_preds = copy.deepcopy(val_preds)
 val_score_fold = f1_score(val_target.argmax(axis=2 ).reshape(( -1)) , best_val_preds, average='macro')
 models[k] = best_model
print("BEST VALIDATION SCORE(F1): ", val_score_fold)
 local_val_score +=(1/n_folds)* val_score_fold
print("Final Score ", local_val_score )<choose_model_class>
grouped = train.groupby('Ticket')
 k = 0
for name, group in grouped:
if(len(grouped.get_group(name)) > 1):
print(group.loc[:,['Survived','Name', 'Fare']])
 k += 1
if(k>10):
break
for k in range(n_folds):
test_p = np.zeros(( int(test_input.shape[0] * test_input.shape[1])))
 k += 1
model = Bi_RNN(lstm_input_size, hidden_state_size, batch_size=batch_size, output_dim=output_dim, num_layers=num_sequence_layers, rnn_type=rnn_type)
 model = model.to(device)
 model.load_state_dict(models[k])
 model.eval()
for i, x_batch in enumerate(test_loader):
x_batch = x_batch.view(-1, num_time_steps, lstm_input_size)
 y_pred = model(x_batch.cuda() ).detach()
pred = F.softmax(y_pred, 2 ).detach().cpu().numpy().argmax(axis=-1)
 test_p[i * batch_size * test_input.shape[1]:(i + 1)* batch_size * test_input.shape[1]] = pred.reshape(( -1))
del y_pred, x_batch, pred
test_preds +=(1/n_folds)* test_p<save_to_csv>
dummy = mosaic(train,["Survived","Sex","Pclass"] )
df_sub = pd.read_csv("/kaggle/input/liverpool-ion-switching/sample_submission.csv", dtype = {'time': str})
 df_sub.open_channels = np.array(test_preds, np.int)
 df_sub.to_csv("submission_bilstm.csv", index=False )<install_modules>
print(train[train['Embarked'].isnull() ] )
!pip install --no-warn-conflicts -q tensorflow-addons<import_modules>
combine.where(( combine['Embarked'] !='Q')&(combine['Pclass'] < 1.5)& \ 
(combine['Sex'] == "female")).groupby(['Embarked','Pclass','Sex','Parch','SibSp'] ).size()
Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,
RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline<define_variables>
train['Embarked'].iloc[61] = "C"
train['Embarked'].iloc[829] = "C"
EPOCHS=125
NNBATCHSIZE=16
BATCHSIZE = 4000
SERIESIZE = 500000
SEED = 321
SELECT = True
SPLITS = 5
LR = 0.001
fe_config = [ 
(True, 4000),
]<define_variables>
print(test[test['Fare'].isnull() ] )
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
<load_from_csv>
print(test[test['Fare'].isnull() ] )
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('/kaggle/input/detrendedwithkalman/train_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/detrendedwithkalman/test_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub 
<data_type_conversions>
test['Fare'].iloc[152] = combine['Fare'][combine['Pclass'] == 3].dropna().median()
print(test['Fare'].iloc[152] )
def batching(df : pd.DataFrame,
batch_size : int,
colname: Text)-> pd.DataFrame :
df[colname] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df[colname] = df[colname].astype(np.uint16)
 return df 
<create_dataframe>
combine = pd.concat([train.drop('Survived',1),test])
 survived = train['Survived']
combine['Child'] = combine['Age']<=10
combine['Cabin_known'] = combine['Cabin'].isnull() == False
combine['Age_known'] = combine['Age'].isnull() == False
combine['Family'] = combine['SibSp'] + combine['Parch']
combine['Alone'] =(combine['SibSp'] + combine['Parch'])== 0
combine['Large_Family'] =(combine['SibSp']>2)|(combine['Parch']>3)
 combine['Deck'] = combine['Cabin'].str[0]
combine['Deck'] = combine['Deck'].fillna(value='U')
 combine['Ttype'] = combine['Ticket'].str[0]
combine['Title'] = combine['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
combine['Fare_cat'] = pd.DataFrame(np.floor(np.log10(combine['Fare'] + 1)) ).astype('int')
 combine['Bad_ticket'] = combine['Ttype'].isin(['3','4','5','6','7','8','A','L','W'])
 combine['Young'] =(combine['Age']<=30)|(combine['Title'].isin(['Master','Miss','Mlle']))
combine['Shared_ticket'] = np.where(combine.groupby('Ticket')['Name'].transform('count')> 1, 1, 0)
 combine['Ticket_group'] = combine.groupby('Ticket')['Name'].transform('count')
 combine['Fare_eff'] = combine['Fare']/combine['Ticket_group']
combine['Fare_eff_cat'] = np.where(combine['Fare_eff']>16.0, 2, 1)
 combine['Fare_eff_cat'] = np.where(combine['Fare_eff']<8.5,0,combine['Fare_eff_cat'])
 test = combine.iloc[len(train):]
train = combine.iloc[:len(train)]
train['Survived'] = survived
surv = train[train['Survived']==1]
nosurv = train[train['Survived']==0]
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby('group')['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df.groupby('group')['signal'].shift(-1*shift_size ).fillna(0)
 if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
for shift_size in shift_sizes:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(-1*shift_size ).fillna(0)
 return df 
<create_dataframe>
print(combine['Age'].groupby(combine['Title'] ).count())
 print(combine['Age'].groupby(combine['Title'] ).mean())
 print("There are %i unique titles in total."%(len(combine['Title'].unique())) )
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
batch_size : int)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, colname='group')
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3], add_pct_change=False, add_pct_change_lag=False)
 df['signal_2'] = df['signal'] ** 2
return df 
<feature_engineering>
pd.DataFrame(np.floor(np.log10(train['Fare'] + 1)) ).astype('int' ).head(5 )
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame)-> Tuple[pd.DataFrame , pd.DataFrame, List]:
use_cols = [col for col in df.columns if col not in ['index','group', 'open_channels', 'time', 'batch']]
df = df.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 for col in use_cols:
col_mean = pd.concat([df[col], df_test[col]], axis=0 ).mean()
df[col] = df[col].fillna(col_mean)
 df_test[col] = df_test[col].fillna(col_mean)
 gc.collect()
return df, df_test, use_cols 
<concatenate>
combine.iloc[np.where(combine.groupby('Ticket')['Fare'].transform('std')> 0)]
def augment(X: np.array, y:np.array)-> Tuple[np.array, np.array]:
X = np.vstack(( X, np.flip(X, axis=1)))
 y = np.vstack(( y, np.flip(y, axis=1)))
 return X, y<create_dataframe>
combine[(combine['Pclass']==1)&(combine['Fare_eff']>0)&(combine['Fare_eff']<10)]
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
batch_col : Text,
feats : List,
sample_submission: pd.DataFrame,
nn_epochs : int,
nn_batch_size : int)-> NoReturn:
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
train_x, train_y = augment(train_x, train_y)
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 cb_prg = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,leave_overall_progress=False, show_epoch_progress=False,show_overall_progress=True)
 model.fit(train_x,train_y,
epochs=nn_epochs,
callbacks=[cb_prg, cb_lr_schedule, MacroF1(model, valid_x,valid_y)],
batch_size=nn_batch_size,verbose=0,
validation_data=(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ =f1_score(np.argmax(train_tr, axis=2 ).reshape(-1), np.argmax(oof_, axis=1), average = 'macro')
 logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis=1 ).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return 
<set_options>
combine[(combine['Pclass']==3)&(np.log10(combine['Fare_eff'])>1.2)]
def lr_schedule(epoch):
if epoch < 40:
lr = LR
elif epoch < 50:
lr = LR / 3
elif epoch < 60:
lr = LR / 6
elif epoch < 75:
lr = LR / 9
elif epoch < 85:
lr = LR / 12
elif epoch < 100:
lr = LR / 15
elif epoch < 110:
lr = LR / 25
else:
lr = LR / 50
return lr<normalization>
combine = pd.concat([train.drop('Survived',1),test])
 survived = train['Survived']
combine["Sex"] = combine["Sex"].astype("category")
 combine["Sex"].cat.categories = [0,1]
combine["Sex"] = combine["Sex"].astype("int")
 combine["Embarked"] = combine["Embarked"].astype("category")
 combine["Embarked"].cat.categories = [0,1,2]
combine["Embarked"] = combine["Embarked"].astype("int")
 combine["Deck"] = combine["Deck"].astype("category")
 combine["Deck"].cat.categories = [0,1,2,3,4,5,6,7,8]
combine["Deck"] = combine["Deck"].astype("int")
 test = combine.iloc[len(train):]
train = combine.iloc[:len(train)]
train['Survived'] = survived
train.loc[:,["Sex","Embarked"]].head()
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 get_custom_objects().update({'mish': Activation(mish)})
<compute_train_metric>
training, testing = train_test_split(train, test_size=0.2, random_state=0)
 print("Total sample size = %i; training sample size = %i, testing sample size = %i"\
%(train.shape[0],training.shape[0],testing.shape[0]))
def categorical_focal_loss(gamma=2.0, alpha=0.25):

 def focal_loss(y_true, y_pred):
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)
 cross_entropy = -y_true*K.log(y_pred)
 weight = alpha * y_true * K.pow(( 1-y_pred), gamma)
 loss = weight * cross_entropy
loss = K.sum(loss, axis=1)
 return loss
return focal_loss<choose_model_class>
cols = ['Sex','Pclass','Cabin_known','Large_Family','Parch',
'SibSp','Young','Alone','Shared_ticket','Child']
tcols = np.append(['Survived'],cols)
 df = training.loc[:,tcols].dropna()
X = df.loc[:,cols]
y = np.ravel(df.loc[:,['Survived']] )
def WaveNetResidualConv1D(num_filters, kernel_size, stacked_layer):
def build_residual_block(l_input):
resid_input = l_input
for dilation_rate in [2**i for i in range(stacked_layer)]:
l_sigmoid_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='sigmoid' )(l_input)
 l_tanh_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='mish' )(l_input)
 l_input = Multiply()([l_sigmoid_conv1d, l_tanh_conv1d])
 l_input = Conv1D(num_filters, 1, padding='same' )(l_input)
 resid_input = Add()([resid_input ,l_input])
 return resid_input
return build_residual_block
def Classifier(shape_):
num_filters_ = 16
kernel_size_ = 4
stacked_layers_ = [12, 8, 4, 1]
l_input = Input(shape=(shape_))
x = Conv1D(num_filters_, 1, padding='same' )(l_input)
 x = WaveNetResidualConv1D(num_filters_, kernel_size_, stacked_layers_[0] )(x)
 x = Conv1D(num_filters_*2, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*2, kernel_size_, stacked_layers_[1] )(x)
 x = Conv1D(num_filters_*4, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*4, kernel_size_, stacked_layers_[2] )(x)
 x = Conv1D(num_filters_*8, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*8, kernel_size_, stacked_layers_[3] )(x)
 l_output = Dense(11, activation='softmax' )(x)
 model = models.Model(inputs=[l_input], outputs=[l_output])
 opt = Adam(lr=LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss=categorical_focal_loss(gamma=3.0, alpha=0.25), optimizer=opt, metrics=['categorical_accuracy'])
 return model<predict_on_test>
clf_log = LogisticRegression()
clf_log = clf_log.fit(X,y)
 score_log = clf_log.score(X,y)
 print(score_log )
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis=2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis=2 ).reshape(-1)
 score = f1_score(self.targets, pred, average="macro")
 print(f' F1Macro: {score:.5f}')
<normalization>
pd.DataFrame(list(zip(X.columns, np.transpose(clf_log.coef_))))
class GaussRankScaler() :
def __init__(self):
self.epsilon = 0.001
self.lower = -1 + self.epsilon
self.upper = 1 - self.epsilon
self.range = self.upper - self.lower
def fit_transform(self, X):
	
i = np.argsort(X, axis = 0)
 j = np.argsort(i, axis = 0)
 assert(j.min() == 0 ).all()
assert(j.max() == len(j)- 1 ).all()
j_range = len(j)- 1
self.divider = j_range / self.range
transformed = j / self.divider
transformed = transformed - self.upper
transformed = erfinv(transformed)
 return transformed 
<normalization>
cols = ['Sex','Pclass','Cabin_known','Large_Family','Shared_ticket','Young','Alone','Child']
tcols = np.append(['Survived'],cols)
 df = training.loc[:,tcols].dropna()
X = df.loc[:,cols]
y = np.ravel(df.loc[:,['Survived']])
 df_test = testing.loc[:,tcols].dropna()
X_test = df_test.loc[:,cols]
y_test = np.ravel(df_test.loc[:,['Survived']] )
def rankgaus_transform(train, test):
shape_ = train.shape[0]
data = pd.concat([train[['signal']], test[['signal']]] ).values
rgsc = GaussRankScaler()
data = rgsc.fit_transform(data)
 train['signal'] = data[:shape_]
test['signal'] = data[shape_:]
return train, test<normalization>
clf_log = LogisticRegression()
clf_log = clf_log.fit(X,y)
 score_log = cross_val_score(clf_log, X, y, cv=5 ).mean()
print(score_log )
def scale(train, test):
sc = StandardScaler()
train['signal'] = sc.fit_transform(train.signal.values.reshape(-1, 1))
test['signal'] = sc.transform(test.signal.values.reshape(-1, 1))
return train, test<define_variables>
clf_pctr = Perceptron(
 class_weight='balanced' 
)
 clf_pctr = clf_pctr.fit(X,y)
 score_pctr = cross_val_score(clf_pctr, X, y, cv=5 ).mean()
print(score_pctr )
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 train = batching(train, batch_size=SERIESIZE, colname='batch')
 selection = [0,2,3,4,5]
train = train.loc[train['batch'].isin(selection)]
train = train.reset_index(drop=True)
 train = train.drop('batch', axis=1)
 train, test = scale(train, test)
 logger.info('Reading and Normalizing Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], batch_size=config[1])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], batch_size=config[1])
 train, test, feats = feature_selection(train, test)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running Wavenet model'):
logger.info(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, splits=SPLITS, batch_col='group', feats=feats, sample_submission=sample_submission, nn_epochs=EPOCHS, nn_batch_size=NNBATCHSIZE)
 logger.info(f'Training completed...')
<install_modules>
clf_knn = KNeighborsClassifier(
 n_neighbors=10,
weights='distance' 
)
 clf_knn = clf_knn.fit(X,y)
 score_knn = cross_val_score(clf_knn, X, y, cv=5 ).mean()
print(score_knn )
!pip install --no-warn-conflicts -q tensorflow-addons<import_modules>
clf_svm = svm.SVC(
 class_weight='balanced' 
)
 clf_svm.fit(X, y)
 score_svm = cross_val_score(clf_svm, X, y, cv=5 ).mean()
print(score_svm )
Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,
RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline<define_variables>
clf_bay = GaussianNB()
clf_bay.fit(X,y)
 score_bay = cross_val_score(clf_bay, X, y, cv=5 ).mean()
print(score_bay )
EPOCHS=125
NNBATCHSIZE=16
BATCHSIZE = 4000
SERIESIZE = 500000
SEED = 321
SELECT = True
SPLITS = 5
LR = 0.001
fe_config = [ 
(True, 4000),
]<define_variables>
bagging = BaggingClassifier(
 KNeighborsClassifier(
 n_neighbors=2,
weights='distance' 
),
oob_score=True,
max_samples=0.5,
max_features=1.0 
)
 clf_bag = bagging.fit(X,y)
 score_bag = clf_bag.oob_score_
print(score_bag )
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
<load_from_csv>
clf_tree = tree.DecisionTreeClassifier(
 class_weight="balanced",\
min_weight_fraction_leaf=0.01\ 
)
 clf_tree = clf_tree.fit(X,y)
 score_tree = cross_val_score(clf_tree, X, y, cv=5 ).mean()
print(score_tree )
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('/kaggle/input/detrendedwithkalman/train_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/detrendedwithkalman/test_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub 
<data_type_conversions>
clf_rf = RandomForestClassifier(
 n_estimators=1000, \
max_depth=None, \
min_samples_split=10 \ 
)
 clf_rf = clf_rf.fit(X,y)
 score_rf = cross_val_score(clf_rf, X, y, cv=5 ).mean()
print(score_rf )
def batching(df : pd.DataFrame,
batch_size : int,
colname: Text)-> pd.DataFrame :
df[colname] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df[colname] = df[colname].astype(np.uint16)
 return df 
<create_dataframe>
clf_ext = ExtraTreesClassifier(
 max_features='auto',
bootstrap=True,
oob_score=True,
n_estimators=1000,
max_depth=None,
min_samples_split=10 
)
 clf_ext = clf_ext.fit(X,y)
 score_ext = cross_val_score(clf_ext, X, y, cv=5 ).mean()
print(score_ext )
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby('group')['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df.groupby('group')['signal'].shift(-1*shift_size ).fillna(0)
 if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
for shift_size in shift_sizes:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(-1*shift_size ).fillna(0)
 return df 
<create_dataframe>
warnings.filterwarnings("ignore")
 clf_gb = GradientBoostingClassifier(
 n_estimators=1000,
learning_rate=0.1,
max_depth=3,
subsample=0.5,
random_state=0 ).fit(X, y)
 clf_gb.fit(X,y)
 score_gb = cross_val_score(clf_gb, X, y, cv=5 ).mean()
print(score_gb )
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
batch_size : int)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, colname='group')
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3], add_pct_change=False, add_pct_change_lag=False)
 df['signal_2'] = df['signal'] ** 2
return df 
<feature_engineering>
clf_ada = AdaBoostClassifier(n_estimators=400, learning_rate=0.1)
 clf_ada.fit(X,y)
 score_ada = cross_val_score(clf_ada, X, y, cv=5 ).mean()
print(score_ada )
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame)-> Tuple[pd.DataFrame , pd.DataFrame, List]:
use_cols = [col for col in df.columns if col not in ['index','group', 'open_channels', 'time', 'batch']]
df = df.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 for col in use_cols:
col_mean = pd.concat([df[col], df_test[col]], axis=0 ).mean()
df[col] = df[col].fillna(col_mean)
 df_test[col] = df_test[col].fillna(col_mean)
 gc.collect()
return df, df_test, use_cols 
<concatenate>
clf_xgb = xgb.XGBClassifier(
 max_depth=2,
n_estimators=500,
subsample=0.5,
learning_rate=0.1 
)
 clf_xgb.fit(X,y)
 score_xgb = cross_val_score(clf_xgb, X, y, cv=5 ).mean()
print(score_xgb )
def augment(X: np.array, y:np.array)-> Tuple[np.array, np.array]:
X = np.vstack(( X, np.flip(X, axis=1)))
 y = np.vstack(( y, np.flip(y, axis=1)))
 return X, y<create_dataframe>
clf_lgb = lgb.LGBMClassifier(
 max_depth=2,
n_estimators=500,
subsample=0.5,
learning_rate=0.1 
)
 clf_lgb.fit(X,y)
 score_lgb = cross_val_score(clf_lgb, X, y, cv=5 ).mean()
print(score_lgb )
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
batch_col : Text,
feats : List,
sample_submission: pd.DataFrame,
nn_epochs : int,
nn_batch_size : int)-> NoReturn:
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
train_x, train_y = augment(train_x, train_y)
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 cb_prg = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,leave_overall_progress=False, show_epoch_progress=False,show_overall_progress=True)
 model.fit(train_x,train_y,
epochs=nn_epochs,
callbacks=[cb_prg, cb_lr_schedule, MacroF1(model, valid_x,valid_y)],
batch_size=nn_batch_size,verbose=0,
validation_data=(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ =f1_score(np.argmax(train_tr, axis=2 ).reshape(-1), np.argmax(oof_, axis=1), average = 'macro')
 logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis=1 ).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return 
<set_options>
clf_ext = ExtraTreesClassifier(max_features='auto',bootstrap=True,oob_score=True)
 param_grid = { "criterion" : ["gini", "entropy"],
"min_samples_leaf" : [1, 5, 10],
"min_samples_split" : [8, 10, 12],
"n_estimators": [20, 50, 100]}
gs = GridSearchCV(estimator=clf_ext, param_grid=param_grid, scoring='accuracy', cv=3)
 gs = gs.fit(X,y)
 print(gs.best_score_)
 print(gs.best_params_ )
def lr_schedule(epoch):
if epoch < 40:
lr = LR
elif epoch < 50:
lr = LR / 3
elif epoch < 60:
lr = LR / 6
elif epoch < 75:
lr = LR / 9
elif epoch < 85:
lr = LR / 12
elif epoch < 100:
lr = LR / 15
elif epoch < 110:
lr = LR / 25
else:
lr = LR / 50
return lr<normalization>
clf_ext = ExtraTreesClassifier(
 max_features='auto',
bootstrap=True,
oob_score=True,
criterion='gini',
min_samples_leaf=5,
min_samples_split=8,
n_estimators=50 
)
 clf_ext = clf_ext.fit(X,y)
 score_ext = clf_ext.score(X,y)
 print(score_ext)
 pd.DataFrame(list(zip(X.columns, np.transpose(clf_ext.feature_importances_)))\
).sort_values(1, ascending=False )
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 get_custom_objects().update({'mish': Activation(mish)})
<compute_train_metric>
clf = clf_ext
scores = cross_val_score(clf, X, y, cv=5)
 print(scores)
 print("Mean score = %.3f, Std deviation = %.3f"%(np.mean(scores),np.std(scores)) )
def categorical_focal_loss(gamma=2.0, alpha=0.25):

 def focal_loss(y_true, y_pred):
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)
 cross_entropy = -y_true*K.log(y_pred)
 weight = alpha * y_true * K.pow(( 1-y_pred), gamma)
 loss = weight * cross_entropy
loss = K.sum(loss, axis=1)
 return loss
return focal_loss<choose_model_class>
score_ext_test = clf_ext.score(X_test,y_test)
 print(score_ext_test )
def WaveNetResidualConv1D(num_filters, kernel_size, stacked_layer):
def build_residual_block(l_input):
resid_input = l_input
for dilation_rate in [2**i for i in range(stacked_layer)]:
l_sigmoid_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='sigmoid' )(l_input)
 l_tanh_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='mish' )(l_input)
 l_input = Multiply()([l_sigmoid_conv1d, l_tanh_conv1d])
 l_input = Conv1D(num_filters, 1, padding='same' )(l_input)
 resid_input = Add()([resid_input ,l_input])
 return resid_input
return build_residual_block
def Classifier(shape_):
num_filters_ = 16
kernel_size_ = 4
stacked_layers_ = [12, 8, 4, 1]
l_input = Input(shape=(shape_))
x = Conv1D(num_filters_, 1, padding='same' )(l_input)
 x = WaveNetResidualConv1D(num_filters_, kernel_size_, stacked_layers_[0] )(x)
 x = Conv1D(num_filters_*2, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*2, kernel_size_, stacked_layers_[1] )(x)
 x = Conv1D(num_filters_*4, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*4, kernel_size_, stacked_layers_[2] )(x)
 x = Conv1D(num_filters_*8, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*8, kernel_size_, stacked_layers_[3] )(x)
 l_output = Dense(11, activation='softmax' )(x)
 model = models.Model(inputs=[l_input], outputs=[l_output])
 opt = Adam(lr=LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss=categorical_focal_loss(gamma=3.0, alpha=0.25), optimizer=opt, metrics=['categorical_accuracy'])
 return model<predict_on_test>
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Gradient Boosting', 'Bagging KNN',
'Decision Tree','XGBoost','LightGBM','ExtraTree','Perceptron', 'Naive Bayes'],
'Score': [score_svm, score_knn, score_log, score_rf, score_gb, score_bag,
score_tree,score_xgb,score_lgb,score_ext,score_pctr, score_bay]})
 models.sort_values(by='Score', ascending=False )
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis=2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis=2 ).reshape(-1)
 score = f1_score(self.targets, pred, average="macro")
 print(f' F1Macro: {score:.5f}')
<normalization>
clf_vote = VotingClassifier(
 estimators=[ 
('knn', clf_knn), 
('svm', clf_svm), 
('extra', clf_ext), 
('xgb', clf_xgb), 
('percep', clf_pctr), 
('logistic', clf_log),
],
weights=[2,2,3,3,1,2],
voting='hard')
 clf_vote.fit(X,y)
 scores = cross_val_score(clf_vote, X, y, cv=5, scoring='accuracy')
 print("Voting: Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std()))
class GaussRankScaler() :
def __init__(self):
self.epsilon = 0.001
self.lower = -1 + self.epsilon
self.upper = 1 - self.epsilon
self.range = self.upper - self.lower
def fit_transform(self, X):
	
i = np.argsort(X, axis = 0)
 j = np.argsort(i, axis = 0)
 assert(j.min() == 0 ).all()
assert(j.max() == len(j)- 1 ).all()
j_range = len(j)- 1
self.divider = j_range / self.range
transformed = j / self.divider
transformed = transformed - self.upper
transformed = erfinv(transformed)
 return transformed 
<normalization>
train = X
ntrain = train.shape[0]
ntest = test.shape[0]
SEED = 0
NFOLDS = 5
kf = KFold(n_splits=NFOLDS, random_state=SEED)
 class SklearnHelper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y )
def rankgaus_transform(train, test):
shape_ = train.shape[0]
data = pd.concat([train[['signal']], test[['signal']]] ).values
rgsc = GaussRankScaler()
data = rgsc.fit_transform(data)
 train['signal'] = data[:shape_]
test['signal'] = data[shape_:]
return train, test<normalization>
def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
x_tr = x_train[train_index]
y_tr = y_train[train_index]
x_te = x_train[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = np.median(oof_test_skf,axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
def scale(train, test):
sc = StandardScaler()
train['signal'] = sc.fit_transform(train.signal.values.reshape(-1, 1))
test['signal'] = sc.transform(test.signal.values.reshape(-1, 1))
return train, test<define_variables>
rf_params = {
'n_estimators': 500,
'warm_start': True,
'max_depth': 6,
'min_samples_leaf': 2,
'max_features' : 'sqrt',
}
et_params = {
'n_estimators':500,
'max_depth': 8,
'min_samples_leaf': 2,
}
ada_params = {
'n_estimators': 500,
'learning_rate' : 0.75
}
gb_params = {
'n_estimators': 500,
'max_depth': 5,
'min_samples_leaf': 2,
}
svc_params = {
'kernel' : 'linear',
'C' : 0.025
}
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 train = batching(train, batch_size=SERIESIZE, colname='batch')
 selection = [0,2,3,4,5]
train = train.loc[train['batch'].isin(selection)]
train = train.reset_index(drop=True)
 train = train.drop('batch', axis=1)
 train, test = scale(train, test)
 logger.info('Reading and Normalizing Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], batch_size=config[1])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], batch_size=config[1])
 train, test, feats = feature_selection(train, test)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running Wavenet model'):
logger.info(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, splits=SPLITS, batch_col='group', feats=feats, sample_submission=sample_submission, nn_epochs=EPOCHS, nn_batch_size=NNBATCHSIZE)
 logger.info(f'Training completed...')
<load_from_csv>
rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)
 et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)
 ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)
 gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)
 svc = SklearnHelper(clf=svm.SVC, seed=SEED, params=svc_params )
def read_data() :
print('Reading training, testing and submission data...')
 train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv')
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv')
 submission = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time':str})
 print('Train set has {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Test set has {} rows and {} columns'.format(test.shape[0], test.shape[1]))
return train, test, submission
def get_batch(train, test):
batch = 50
total_batches = 14
train['set'] = 'train'
test['set'] = 'test'
data = pd.concat([train, test])
 for i in range(int(total_batches)) :
data.loc[(data['time'] > i * batch)&(data['time'] <=(i + 1)* batch), 'batch'] = i + 1
train = data[data['set'] == 'train']
test = data[data['set'] == 'test']
train.drop(['set'], inplace = True, axis = 1)
 test.drop(['set'], inplace = True, axis = 1)
 del data
return train, test
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
if col!='open_channels':
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<categorify>
y_train = y
train = X
foo = test.loc[:,cols]
x_train = train.values
x_test = foo.values
def rolling_features(train, test):
pre_train = train.copy()
pre_test = test.copy()
for df in [pre_train, pre_test]:
df['lag_t1'] = df.groupby('batch')['signal'].transform(lambda x: x.shift(1))
df['lag_t2'] = df.groupby('batch')['signal'].transform(lambda x: x.shift(2))
df['lag_t3'] = df.groupby('batch')['signal'].transform(lambda x: x.shift(3))
df['lead_t1'] = df.groupby('batch')['signal'].transform(lambda x: x.shift(-1))
df['lead_t2'] = df.groupby('batch')['signal'].transform(lambda x: x.shift(-2))
df['lead_t3'] = df.groupby('batch')['signal'].transform(lambda x: x.shift(-3))
for window in [1000, 5000, 10000, 20000, 40000, 80000]:
df['signalmean_t' + str(window)] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(1 ).rolling(window ).mean())
 df['signalstd_t' + str(window)] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(1 ).rolling(window ).std())
 df['signalvar_t' + str(window)] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(1 ).rolling(window ).var())
 df['signalmin_t' + str(window)] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(1 ).rolling(window ).min())
 df['signalmax_t' + str(window)] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(1 ).rolling(window ).max())
 min_max =(df['signal'] - df['signalmin_t' + str(window)])/(df['signalmax_t' + str(window)] - df['signalmin_t' + str(window)])
 df['norm_t' + str(window)] = min_max *(np.floor(df['signalmax_t' + str(window)])- np.ceil(df['signalmin_t' + str(window)]))
df['signalmean_t' + str(window)+ '_lead'] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(- window - 1 ).rolling(window ).mean())
 df['signalstd_t' + str(window)+ '_lead'] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(- window - 1 ).rolling(window ).std())
 df['signalvar_t' + str(window)+ '_lead'] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(- window - 1 ).rolling(window ).var())
 df['signalmin_t' + str(window)+ '_lead'] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(- window - 1 ).rolling(window ).min())
 df['signalmax_t' + str(window)+ '_lead'] = df.groupby(['batch'])['signal'].transform(lambda x: x.shift(- window - 1 ).rolling(window ).max())
 min_max =(df['signal'] - df['signalmin_t' + str(window)+ '_lead'])/(df['signalmax_t' + str(window)+ '_lead'] - df['signalmin_t' + str(window)+ '_lead'])
 df['norm_t' + str(window)+ '_lead'] = min_max *(np.floor(df['signalmax_t' + str(window)+ '_lead'])- np.ceil(df['signalmin_t' + str(window)+ '_lead']))
del train, test, min_max
return pre_train, pre_test
def static_batch_features(df, n):
df = df.copy()
df.drop('batch', inplace = True, axis = 1)
 df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10000)- 1 ).values
df['batch_' + str(n)] = df.index // n
df['batch_index_' + str(n)] = df.index -(df['batch_' + str(n)] * n)
 df['batch_slices_' + str(n)] = df['batch_index_' + str(n)] //(n / 10)
 df['batch_slices2_' + str(n)] = df.apply(lambda r: '_'.join([str(r['batch_' + str(n)] ).zfill(3), str(r['batch_slices_' + str(n)] ).zfill(3)]), axis=1)
 for c in ['batch_' + str(n), 'batch_slices2_' + str(n)]:
d = {}
d['mean' + c] = df.groupby([c])['signal'].mean()
d['median' + c] = df.groupby([c])['signal'].median()
d['max' + c] = df.groupby([c])['signal'].max()
d['min' + c] = df.groupby([c])['signal'].min()
d['std' + c] = df.groupby([c])['signal'].std()
d['p10' + c] = df.groupby([c])['signal'].apply(lambda x: np.percentile(x, 10))
d['p25' + c] = df.groupby([c])['signal'].apply(lambda x: np.percentile(x, 25))
d['p75' + c] = df.groupby([c])['signal'].apply(lambda x: np.percentile(x, 75))
d['p90' + c] = df.groupby([c])['signal'].apply(lambda x: np.percentile(x, 90))
d['skew' + c] = df.groupby([c])['signal'].apply(lambda x: pd.Series(x ).skew())
 d['kurtosis' + c] = df.groupby([c])['signal'].apply(lambda x: pd.Series(x ).kurtosis())
 min_max =(d['mean' + c] - d['min' + c])/(d['max' + c] - d['min' + c])
 d['norm' + c] = min_max *(np.floor(d['max' + c])- np.ceil(d['min' + c]))
d['mean_abs_chg' + c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max' + c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min' + c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range' + c] = d['max' + c] - d['min' + c]
d['maxtomin' + c] = d['max' + c] / d['min' + c]
d['abs_avg' + c] =(d['abs_min' + c] + d['abs_max' + c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels', 'batch', 'batch_' + str(n),
'batch_index_' + str(n), 'batch_slices_' + str(n),
'batch_slices2_' + str(n)]]:
df[c + '_msignal'] = df[c] - df['signal']
df.reset_index(drop = True, inplace = True)
 return df<prepare_x_and_y>
et_oof_train, et_oof_test = get_oof(et, x_train, y_train, x_test)
 rf_oof_train, rf_oof_test = get_oof(rf,x_train, y_train, x_test)
 ada_oof_train, ada_oof_test = get_oof(ada, x_train, y_train, x_test)
 gb_oof_train, gb_oof_test = get_oof(gb,x_train, y_train, x_test)
 svc_oof_train, svc_oof_test = get_oof(svc,x_train, y_train, x_test)
 print("Training is complete" )
def run_lgb(pre_train, pre_test, features, params):
kf = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 42)
 target = 'open_channels'
oof_pred = np.zeros(len(pre_train))
y_pred = np.zeros(len(pre_test))
for fold,(tr_ind, val_ind)in enumerate(kf.split(pre_train, pre_train[target])) :
x_train, x_val = pre_train[features].iloc[tr_ind], pre_train[features].iloc[val_ind]
y_train, y_val = pre_train[target][tr_ind], pre_train[target][val_ind]
train_set = lgb.Dataset(x_train, y_train)
 val_set = lgb.Dataset(x_val, y_val)
 model = lgb.train(params, train_set, num_boost_round = 10000, early_stopping_rounds = 50,
valid_sets = [train_set, val_set], verbose_eval = 100)
 oof_pred[val_ind] = model.predict(x_val)
 y_pred += model.predict(pre_test[features])/ kf.n_splits
rmse_score = np.sqrt(metrics.mean_squared_error(pre_train[target], oof_pred))
oof_pred = np.round(np.clip(oof_pred, 0, 10)).astype(int)
 round_y_pred = np.round(np.clip(y_pred, 0, 10)).astype(int)
 f1 = metrics.f1_score(pre_train[target], oof_pred, average = 'macro')
 print(f'Our oof rmse score is {rmse_score}')
 print(f'Our oof macro f1 score is {f1}')
 return round_y_pred<choose_model_class>
base_predictions_train = pd.DataFrame({'RandomForest': rf_oof_train.ravel() ,
'ExtraTrees': et_oof_train.ravel() ,
'AdaBoost': ada_oof_train.ravel() ,
'SVM' : svc_oof_train.ravel() ,
'GradientBoost': gb_oof_train.ravel()
})
 base_predictions_train.head()
def Kalman1D(observations,damping=1):
observation_covariance = damping
initial_value_guess = observations[0]
transition_matrix = 1
transition_covariance = 0.1
initial_value_guess
kf = KalmanFilter(
 initial_state_mean=initial_value_guess,
initial_state_covariance=observation_covariance,
observation_covariance=observation_covariance,
transition_covariance=transition_covariance,
transition_matrices=transition_matrix 
)
 pred_state, state_cov = kf.smooth(observations)
 return pred_state<feature_engineering>
x_train = np.concatenate(( et_oof_train, rf_oof_train, ada_oof_train, gb_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( et_oof_test, rf_oof_test, ada_oof_test, gb_oof_test, svc_oof_test), axis=1 )
train, test, submission = read_data()
observation_covariance =.0015
train['signal'] = Kalman1D(train.signal.values,observation_covariance)
 test['signal'] = Kalman1D(test.signal.values,observation_covariance)
 pre_train4 = divide_and_add_features(train['signal'])
 pre_test4 = divide_and_add_features(test['signal'])
 pre_train4.drop(['signal'], inplace = True, axis = 1)
 pre_test4.drop(['signal'], inplace = True, axis = 1)
 pre_train4.reset_index(inplace = True, drop = True)
 pre_test4.reset_index(inplace = True, drop = True)
 pre_train4 = reduce_mem_usage(pre_train4)
 pre_test4 = reduce_mem_usage(pre_test4)
 train, test = get_batch(train, test)
 pre_train1, pre_test1 = rolling_features(train, test)
 pre_train1 = reduce_mem_usage(pre_train1)
 pre_test1 = reduce_mem_usage(pre_test1)
 pre_train2 = static_batch_features(train, 25000)
 pre_train2 = reduce_mem_usage(pre_train2)
 pre_test2 = static_batch_features(test, 25000)
 pre_test2 = reduce_mem_usage(pre_test2)
 del train, test
gc.collect()
feat2 = [col for col in pre_train2.columns if col not in ['open_channels', 'signal', 'time', 'batch_25000',
'batch_index_25000', 'batch_slices_25000', 'batch_slices2_25000']]
pre_train = pd.concat([pre_train1, pre_train2[feat2], pre_train4], axis = 1)
 pre_test = pd.concat([pre_test1, pre_test2[feat2], pre_test4], axis = 1)
 del pre_train1, pre_train2, pre_train4, pre_test1, pre_test2, pre_test4
features = [col for col in pre_train.columns if col not in ['open_channels', 'time', 'batch']]
print('Training with {} features'.format(len(features)))
 params = {'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'num_leaves': 280,
'learning_rate': 0.026623466966581126,
'max_depth': 73,
'lambda_l1': 2.959759088169741,
'lambda_l2': 1.331172832164913,
'bagging_fraction': 0.9655406551472153,
'bagging_freq': 9,
'colsample_bytree': 0.6867118652742716}
round_y_pred = run_lgb(pre_train, pre_test, features, params)
 submission['open_channels'] = round_y_pred
submission.to_csv('submission.csv', index = False )<set_options>
clf_stack = xgb.XGBClassifier(
 n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
scale_pos_weight=1)
 clf_stack = clf_stack.fit(x_train, y_train)
 stack_pred = clf_stack.predict(x_test )
sns.set_style("whitegrid")
<define_variables>
scores = cross_val_score(clf_stack, x_train, y_train, cv=5)
 print(scores)
 print("Mean score = %.3f, Std deviation = %.3f"%(np.mean(scores),np.std(scores)) )
MODEL_TYPE = 'lgb'
TRAINING = True
ENSEMBLE = False
GROUP_BATCH_SIZE = 4000
WINDOWS = [10, 50]
BASE_PATH = '/kaggle/input/liverpool-ion-switching'
DATA_PATH = '/kaggle/input/data-without-drift'
RFC_DATA_PATH = '/kaggle/input/ion-shifted-rfc-proba'
MODELS_PATH = '/kaggle/input/ensemble-models'<save_to_csv>
clf = clf_vote
df2 = test.loc[:,cols].fillna(method='pad')
 surv_pred = clf.predict(df2 )
<feature_engineering><EOS> 
submit = pd.DataFrame({'PassengerId' : test.loc[:,'PassengerId'],
'Survived': stack_pred.T})
 submit.to_csv(".. /working/submit.csv", index=False)