Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<load_from_csv>
!pip install --no-warn-conflicts -q tensorflow-addons
support = pd.read_csv(dfk.iloc[0, 2])
 for k in range(1, 7):
main = pd.read_csv(dfk.iloc[k, 2])
 support = generate(main, support, 0.7)
 sub1 = support<train_on_grid>
Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,
RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline
main = pd.read_csv(dfk.iloc[9, 2])
 sub4 = generate(main, sub3, 0.55)
 drawing1(main, sub3, sub4 )<save_to_csv>
EPOCHS=125
NNBATCHSIZE=16
BATCHSIZE = 4000
SERIESIZE = 500000
SEED = 321
SELECT = True
SPLITS = 5
LR = 0.001
fe_config = [ 
(True, 4000),
]
sub = sub5
sub.to_csv("submission.csv", index=False)
 sub1.to_csv("submission1.csv", index=False)
 sub2.to_csv("submission2.csv", index=False)
 sub3.to_csv("submission3.csv", index=False)
 sub4.to_csv("submission3.csv", index=False)
 !ls<split>
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
target = train.pop('target')
 X_train, X_test, y_train, y_test = train_test_split(train, target, train_size=0.60 )<create_dataframe>
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('/kaggle/input/detrendedwithkalman/train_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/detrendedwithkalman/test_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_valid = lgb.Dataset(X_test, y_test)
 param = {
'seed': 2021,
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'feature_pre_filter': False,
'lambda_l1': 6.540486456085813,
'lambda_l2': 0.01548480538099245,
'num_leaves': 256,
'feature_fraction': 0.52,
'bagging_fraction': 0.6161835249194311,
'bagging_freq': 7,
'min_child_samples': 20,
'learning_rate' : 0.001,
'early_stopping_round' : 1000,
'num_iterations' : 20000
}
lgb_model = lgb.train(param,
lgb_train,
valid_sets=lgb_valid,
num_boost_round=5000,
early_stopping_rounds=100)
<save_to_csv>
def batching(df : pd.DataFrame,
batch_size : int,
colname: Text)-> pd.DataFrame :
df[colname] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df[colname] = df[colname].astype(np.uint16)
 return df
submission['target'] = lgb_model.predict(test)
 submission.to_csv('my_submission.csv' )<import_modules>
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby('group')['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df.groupby('group')['signal'].shift(-1*shift_size ).fillna(0)
 if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
for shift_size in shift_sizes:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(-1*shift_size ).fillna(0)
 return df
import optuna<import_modules>
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
batch_size : int)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, colname='group')
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3], add_pct_change=False, add_pct_change_lag=False)
 df['signal_2'] = df['signal'] ** 2
return df
import xgboost as xgb
from catboost import CatBoostRegressor
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split<load_from_csv>
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame)-> Tuple[pd.DataFrame , pd.DataFrame, List]:
use_cols = [col for col in df.columns if col not in ['index','group', 'open_channels', 'time', 'batch']]
df = df.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 for col in use_cols:
col_mean = pd.concat([df[col], df_test[col]], axis=0 ).mean()
df[col] = df[col].fillna(col_mean)
 df_test[col] = df_test[col].fillna(col_mean)
 gc.collect()
return df, df_test, use_cols
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv' )<define_variables>
def augment(X: np.array, y:np.array)-> Tuple[np.array, np.array]:
X = np.vstack(( X, np.flip(X, axis=1)))
 y = np.vstack(( y, np.flip(y, axis=1)))
 return X, y
columns = [col for col in train.columns.to_list() if col not in ['id','target']]<prepare_x_and_y>
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
batch_col : Text,
feats : List,
sample_submission: pd.DataFrame,
nn_epochs : int,
nn_batch_size : int)-> NoReturn:
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
train_x, train_y = augment(train_x, train_y)
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 cb_prg = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,leave_overall_progress=False, show_epoch_progress=False,show_overall_progress=True)
 model.fit(train_x,train_y,
epochs=nn_epochs,
callbacks=[cb_prg, cb_lr_schedule, MacroF1(model, valid_x,valid_y)],
batch_size=nn_batch_size,verbose=0,
validation_data=(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ =f1_score(np.argmax(train_tr, axis=2 ).reshape(-1), np.argmax(oof_, axis=1), average = 'macro')
 logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis=1 ).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return
data=train[columns]
target=train['target']<split>
def lr_schedule(epoch):
if epoch < 40:
lr = LR
elif epoch < 50:
lr = LR / 3
elif epoch < 60:
lr = LR / 6
elif epoch < 75:
lr = LR / 9
elif epoch < 85:
lr = LR / 12
elif epoch < 100:
lr = LR / 15
elif epoch < 110:
lr = LR / 25
else:
lr = LR / 50
return lr
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02]),
'n_estimators': 4000,
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20]),
'random_state': trial.suggest_categorical('random_state', [24, 48,2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 get_custom_objects().update({'mish': Activation(mish)})
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<create_dataframe>
def categorical_focal_loss(gamma=2.0, alpha=0.25):

 def focal_loss(y_true, y_pred):
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)
 cross_entropy = -y_true*K.log(y_pred)
 weight = alpha * y_true * K.pow(( 1-y_pred), gamma)
 loss = weight * cross_entropy
loss = K.sum(loss, axis=1)
 return loss
return focal_loss
study.trials_dataframe()<init_hyperparams>
def WaveNetResidualConv1D(num_filters, kernel_size, stacked_layer):
def build_residual_block(l_input):
resid_input = l_input
for dilation_rate in [2**i for i in range(stacked_layer)]:
l_sigmoid_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='sigmoid' )(l_input)
 l_tanh_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='mish' )(l_input)
 l_input = Multiply()([l_sigmoid_conv1d, l_tanh_conv1d])
 l_input = Conv1D(num_filters, 1, padding='same' )(l_input)
 resid_input = Add()([resid_input ,l_input])
 return resid_input
return build_residual_block
def Classifier(shape_):
num_filters_ = 16
kernel_size_ = 4
stacked_layers_ = [12, 8, 4, 1]
l_input = Input(shape=(shape_))
x = Conv1D(num_filters_, 1, padding='same' )(l_input)
 x = WaveNetResidualConv1D(num_filters_, kernel_size_, stacked_layers_[0] )(x)
 x = Conv1D(num_filters_*2, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*2, kernel_size_, stacked_layers_[1] )(x)
 x = Conv1D(num_filters_*4, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*4, kernel_size_, stacked_layers_[2] )(x)
 x = Conv1D(num_filters_*8, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*8, kernel_size_, stacked_layers_[3] )(x)
 l_output = Dense(11, activation='softmax' )(x)
 model = models.Model(inputs=[l_input], outputs=[l_output])
 opt = Adam(lr=LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss=categorical_focal_loss(gamma=3.0, alpha=0.25), optimizer=opt, metrics=['categorical_accuracy'])
 return model
Best_trial= {'lambda': 0.0030282073258141168, 'alpha': 0.01563845128469084, 'colsample_bytree': 0.5,
'subsample': 0.7,'n_estimators': 4000, 'learning_rate': 0.01,'max_depth': 15,
'random_state': 2020, 'min_child_weight': 257,'tree_method':'gpu_hist'}<train_model>
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis=2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis=2 ).reshape(-1)
 score = f1_score(self.targets, pred, average="macro")
 print(f' F1Macro: {score:.5f}')
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = xgb.XGBRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1<split>
class GaussRankScaler() :
def __init__(self):
self.epsilon = 0.001
self.lower = -1 + self.epsilon
self.upper = 1 - self.epsilon
self.range = self.upper - self.lower
def fit_transform(self, X):
	
i = np.argsort(X, axis = 0)
 j = np.argsort(i, axis = 0)
 assert(j.min() == 0 ).all()
assert(j.max() == len(j)- 1 ).all()
j_range = len(j)- 1
self.divider = j_range / self.range
transformed = j / self.divider
transformed = transformed - self.upper
transformed = erfinv(transformed)
 return transformed
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'loss_function': 'RMSE',
'task_type': 'GPU',
'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 1e-3, 10.0),
'max_bin': trial.suggest_int('max_bin', 200, 400),
'subsample': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'learning_rate': trial.suggest_uniform('learning_rate', 0.006, 0.018),
'n_estimators': 25000,
'max_depth': trial.suggest_categorical('max_depth', [7,10,14,16]),
'random_state': trial.suggest_categorical('random_state', [24, 48,2020]),
'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 300),
}
model = CatBoostRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=200,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
def rankgaus_transform(train, test):
shape_ = train.shape[0]
data = pd.concat([train[['signal']], test[['signal']]] ).values
rgsc = GaussRankScaler()
data = rgsc.fit_transform(data)
 train['signal'] = data[:shape_]
test['signal'] = data[shape_:]
return train, test
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<init_hyperparams>
def scale(train, test):
sc = StandardScaler()
train['signal'] = sc.fit_transform(train.signal.values.reshape(-1, 1))
test['signal'] = sc.transform(test.signal.values.reshape(-1, 1))
return train, test
Best_trial = {'l2_leaf_reg': 0.02247766515106271, 'max_bin': 364, 'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876, 'max_depth': 10, 'random_state': 24, 'min_data_in_leaf': 300,
'task_type': 'GPU','loss_function': 'RMSE','n_estimators': 25000}<train_model>
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 train = batching(train, batch_size=SERIESIZE, colname='batch')
 selection = [0,2,3,4,5]
train = train.loc[train['batch'].isin(selection)]
train = train.reset_index(drop=True)
 train = train.drop('batch', axis=1)
 train, test = scale(train, test)
 logger.info('Reading and Normalizing Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], batch_size=config[1])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], batch_size=config[1])
 train, test, feats = feature_selection(train, test)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running Wavenet model'):
logger.info(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, splits=SPLITS, batch_col='group', feats=feats, sample_submission=sample_submission, nn_epochs=EPOCHS, nn_batch_size=NNBATCHSIZE)
 logger.info(f'Training completed...')
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=10,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = CatBoostRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=200,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1<init_hyperparams>
!pip install --no-warn-conflicts -q tensorflow-addons
Best_trial = {'l2_leaf_reg': 0.02247766515106271, 'max_bin': 364, 'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876, 'max_depth': 10, 'random_state': 24, 'min_data_in_leaf': 300,
'loss_function': 'RMSE','n_estimators': 25000,'rsm':0.5}<train_model>
Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,
RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=10,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = CatBoostRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1<save_to_csv>
EPOCHS=125
NNBATCHSIZE=16
BATCHSIZE = 4000
SERIESIZE = 500000
SEED = 321
SELECT = True
SPLITS = 5
LR = 0.001
fe_config = [ 
(True, 4000),
]
sub['target']=preds
sub.to_csv('submission.csv', index=False )<set_options>
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
filterwarnings('ignore' )<load_from_csv>
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('/kaggle/input/detrendedwithkalman/train_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/detrendedwithkalman/test_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
train = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv',index_col = 'id')
 print('shape : {} '.format(train.shape))
train.head()<split>
def batching(df : pd.DataFrame,
batch_size : int,
colname: Text)-> pd.DataFrame :
df[colname] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df[colname] = df[colname].astype(np.uint16)
 return df
x = train.drop('target',axis=1)
 y = train['target']
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size = 0.25,random_state = 42 )<choose_model_class>
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby('group')['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df.groupby('group')['signal'].shift(-1*shift_size ).fillna(0)
 if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
for shift_size in shift_sizes:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(-1*shift_size ).fillna(0)
 return df
regressor = xgb.XGBRegressor(
 colsample_bytree=0.5,
alpha=0.01563,
learning_rate=0.01,
max_depth=15,
min_child_weight=257,
n_estimators=4000,
reg_lambda=0.003,
subsample=0.7,
random_state=2020,
metric_period=100,
silent=1)
 regressor.fit(x_train, y_train, early_stopping_rounds=6, eval_set=[(x_test, y_test)], verbose=1 )<load_from_csv>
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
batch_size : int)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, colname='group')
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3], add_pct_change=False, add_pct_change_lag=False)
 df['signal_2'] = df['signal'] ** 2
return df
df_test = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv',index_col = 'id' )<predict_on_test>
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame)-> Tuple[pd.DataFrame , pd.DataFrame, List]:
use_cols = [col for col in df.columns if col not in ['index','group', 'open_channels', 'time', 'batch']]
df = df.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 for col in use_cols:
col_mean = pd.concat([df[col], df_test[col]], axis=0 ).mean()
df[col] = df[col].fillna(col_mean)
 df_test[col] = df_test[col].fillna(col_mean)
 gc.collect()
return df, df_test, use_cols
df_tahmin = pd.DataFrame(regressor.predict(df_test),index= df_test.index,columns = ['target'] )<save_to_csv>
def augment(X: np.array, y:np.array)-> Tuple[np.array, np.array]:
X = np.vstack(( X, np.flip(X, axis=1)))
 y = np.vstack(( y, np.flip(y, axis=1)))
 return X, y
df_tahmin.to_csv('mycsvfile.csv',index=True )<install_modules>
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
batch_col : Text,
feats : List,
sample_submission: pd.DataFrame,
nn_epochs : int,
nn_batch_size : int)-> NoReturn:
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
train_x, train_y = augment(train_x, train_y)
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 cb_prg = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,leave_overall_progress=False, show_epoch_progress=False,show_overall_progress=True)
 model.fit(train_x,train_y,
epochs=nn_epochs,
callbacks=[cb_prg, cb_lr_schedule, MacroF1(model, valid_x,valid_y)],
batch_size=nn_batch_size,verbose=0,
validation_data=(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ =f1_score(np.argmax(train_tr, axis=2 ).reshape(-1), np.argmax(oof_, axis=1), average = 'macro')
 logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis=1 ).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return
!pip install -U git+https://github.com/mljar/mljar-supervised.git@master
!pip install -U matplotlib==3.1.3<import_modules>
def lr_schedule(epoch):
if epoch < 40:
lr = LR
elif epoch < 50:
lr = LR / 3
elif epoch < 60:
lr = LR / 6
elif epoch < 75:
lr = LR / 9
elif epoch < 85:
lr = LR / 12
elif epoch < 100:
lr = LR / 15
elif epoch < 110:
lr = LR / 25
else:
lr = LR / 50
return lr
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from supervised.automl import AutoML 
<load_from_csv>
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 get_custom_objects().update({'mish': Activation(mish)})
train=pd.read_csv(".. /input/tabular-playground-series-jan-2021/train.csv")
 test=pd.read_csv(".. /input/tabular-playground-series-jan-2021/test.csv" )<split>
def categorical_focal_loss(gamma=2.0, alpha=0.25):

 def focal_loss(y_true, y_pred):
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)
 cross_entropy = -y_true*K.log(y_pred)
 weight = alpha * y_true * K.pow(( 1-y_pred), gamma)
 loss = weight * cross_entropy
loss = K.sum(loss, axis=1)
 return loss
return focal_loss
X = train.drop(columns=['target'])
 y = train['target'].values
X_train, X_test, y_train, y_test = train_test_split(
 X,
y,
test_size=0.25,
random_state=123, 
)
<train_model>
def WaveNetResidualConv1D(num_filters, kernel_size, stacked_layer):
def build_residual_block(l_input):
resid_input = l_input
for dilation_rate in [2**i for i in range(stacked_layer)]:
l_sigmoid_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='sigmoid' )(l_input)
 l_tanh_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='mish' )(l_input)
 l_input = Multiply()([l_sigmoid_conv1d, l_tanh_conv1d])
 l_input = Conv1D(num_filters, 1, padding='same' )(l_input)
 resid_input = Add()([resid_input ,l_input])
 return resid_input
return build_residual_block
def Classifier(shape_):
num_filters_ = 16
kernel_size_ = 4
stacked_layers_ = [12, 8, 4, 1]
l_input = Input(shape=(shape_))
x = Conv1D(num_filters_, 1, padding='same' )(l_input)
 x = WaveNetResidualConv1D(num_filters_, kernel_size_, stacked_layers_[0] )(x)
 x = Conv1D(num_filters_*2, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*2, kernel_size_, stacked_layers_[1] )(x)
 x = Conv1D(num_filters_*4, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*4, kernel_size_, stacked_layers_[2] )(x)
 x = Conv1D(num_filters_*8, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*8, kernel_size_, stacked_layers_[3] )(x)
 l_output = Dense(11, activation='softmax' )(x)
 model = models.Model(inputs=[l_input], outputs=[l_output])
 opt = Adam(lr=LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss=categorical_focal_loss(gamma=3.0, alpha=0.25), optimizer=opt, metrics=['categorical_accuracy'])
 return model
automl = AutoML(algorithms=["CatBoost", "Xgboost", "LightGBM","Neural Network"],
model_time_limit=60*60,
start_random_models=10,
hill_climbing_steps=3,
top_models_to_improve=3,
golden_features=True,
features_selection=True,
stack_models=True,
train_ensemble=True,
explain_level=0,
validation_strategy={
"validation_type": "kfold",
"k_folds": 5,
"shuffle": False,
"stratify": True,
})
 automl.fit(X, y)
<predict_on_test>
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis=2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis=2 ).reshape(-1)
 score = f1_score(self.targets, pred, average="macro")
 print(f' F1Macro: {score:.5f}')
predictions = automl.predict(X_test)
 print("Test MSE:", mean_squared_error(y_test, predictions,squared=False))<save_to_csv>
class GaussRankScaler() :
def __init__(self):
self.epsilon = 0.001
self.lower = -1 + self.epsilon
self.upper = 1 - self.epsilon
self.range = self.upper - self.lower
def fit_transform(self, X):
	
i = np.argsort(X, axis = 0)
 j = np.argsort(i, axis = 0)
 assert(j.min() == 0 ).all()
assert(j.max() == len(j)- 1 ).all()
j_range = len(j)- 1
self.divider = j_range / self.range
transformed = j / self.divider
transformed = transformed - self.upper
transformed = erfinv(transformed)
 return transformed
preds = automl.predict(test)
 submission = test[['id']]
submission['target'] = preds
submission.to_csv('submission.csv', index = False )<install_modules>
def rankgaus_transform(train, test):
shape_ = train.shape[0]
data = pd.concat([train[['signal']], test[['signal']]] ).values
rgsc = GaussRankScaler()
data = rgsc.fit_transform(data)
 train['signal'] = data[:shape_]
test['signal'] = data[shape_:]
return train, test
system('pip install pycaret');<import_modules>
def scale(train, test):
sc = StandardScaler()
train['signal'] = sc.fit_transform(train.signal.values.reshape(-1, 1))
test['signal'] = sc.transform(test.signal.values.reshape(-1, 1))
return train, test
import pandas as pd
from pycaret.regression import *<load_from_csv>
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 train = batching(train, batch_size=SERIESIZE, colname='batch')
 selection = [0,2,3,4,5]
train = train.loc[train['batch'].isin(selection)]
train = train.reset_index(drop=True)
 train = train.drop('batch', axis=1)
 train, test = scale(train, test)
 logger.info('Reading and Normalizing Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], batch_size=config[1])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], batch_size=config[1])
 train, test, feats = feature_selection(train, test)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running Wavenet model'):
logger.info(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, splits=SPLITS, batch_col='group', feats=feats, sample_submission=sample_submission, nn_epochs=EPOCHS, nn_batch_size=NNBATCHSIZE)
 logger.info(f'Training completed...')