Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<choose_model_class>
EPOCHS = 5
SPLITS = 6
%%time
def Kalman1D(observations,damping=1):
observation_covariance = damping
initial_value_guess = observations[0]
transition_matrix = 1
transition_covariance = 0.1
initial_value_guess
kf = KalmanFilter(
 initial_state_mean=initial_value_guess,
initial_state_covariance=observation_covariance,
observation_covariance=observation_covariance,
transition_covariance=transition_covariance,
transition_matrices=transition_matrix 
)
 pred_state, state_cov = kf.smooth(observations)
 return pred_state
observation_covariance = 0.0015 
<normalization>
!pip install tensorflow_addons==0.9.1
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def normalize(train, test, col, type_='StandardScaler'):
if type_ == 'StandardScaler':
clf = StandardScaler()
elif type_ == 'MinMaxScaler':
clf = MinMaxScaler()
elif type_ == 'RobustScaler':
clf = RobustScaler()
else:
print('Error type in!')
 train[col] = clf.fit_transform(train[col])
 test[col] = clf.transform(test[col])
 return train, test
train_df, test_df = normalize(train_df, test_df, ['signal'])
 gc.collect()<groupby>
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.0015
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
channel_signal = train_df.groupby('open_channels')['signal'].agg(['min', 'max', np.mean, np.std] ).reset_index()
channel_signal.columns = ['signal_' + i if i!='open_channels' else i for i in channel_signal.columns]
print(channel_signal)
 try:
del channel_signal
except:
print('Variable channel_signal not defined!' )<feature_engineering>
def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index'])
 return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 16, 3, 12)
 x = BatchNormalization()(x)
 x = wave_block(x, 32, 3, 8)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 3:
lr = LR
elif epoch < 10:
lr = LR / 3
elif epoch < 30:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f' )
def process_data(df, windows=window_list, batch_size=20000):
df = df.sort_values('time' ).reset_index(drop=True)
 df.index =(( df.time*10000)- 1 ).values
df['batch'] = df.index // batch_size
df['batch_index'] = df.index % batch_size
df['batch_slices'] = df['batch_index'] //(batch_size//10)
 df['batch_slices1'] = df.apply(lambda x: '_'.join([str(x['batch'] ).zfill(3), str(x['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch', 'batch_slices1']:
d = {}
d[f'min_{c}'] = df.groupby(c)['signal'].min()
d[f'max_{c}'] = df.groupby(c)['signal'].max()
d[f'mean_{c}'] = df.groupby(c)['signal'].mean()
d[f'std_{c}'] = df.groupby(c)['signal'].std()
for per in [10, 25, 75, 90]:
d[f'pct_{per}_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.percentile(x, per))
d[f'skew_{c}'] = df.groupby(c)['signal'].apply(lambda x: pd.Series(x ).skew())
 d[f'kurtosis_{c}'] = df.groupby(c)['signal'].apply(lambda x: pd.Series(x ).kurtosis())
 d[f'mean_abs_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d[f'min_abs_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.min(np.abs(x)))
 d[f'max_abs_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.max(np.abs(x)))
 d[f'range_{c}'] = d[f'max_{c}'] - d[f'min_{c}']
d[f'ratio_{c}'] = d[f'max_{c}'] / d[f'min_{c}']
d[f'avg_abs_{c}'] =(d[f'min_abs_{c}'] + d[f'max_abs_{c}'])/2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['shift_+1'] = [0] + list(df['signal'].values[:-1])
 df['shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['shift_+1'][i] = np.nan
for i in df[df['batch_index']==(batch_size-1)].index:
df['shift_-1'][i] = np.nan
df['shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['shift_-2'] = list(df['signal'].values[:-2])+ [0,] + [1,]
for i in df[df['batch_index'].isin([0, 1])].index:
df['shift_+2'][i] = np.nan
for i in df[df['batch_index'].isin([batch_size-1, batch_size-2])].index:
df['shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices1'])
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels']]:
df[f'{c}_msignal'] = df[f'{c}'] - df['signal']
df = df.replace([np.Inf, -np.Inf], np.nan)
 df.fillna(0, inplace=True)
 gc.collect()
return df
train_df = process_data(train_df)
 test_df = process_data(test_df )<drop_column>
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...' )
%%time
train_df = reduce_mem_usage(train_df)
 test_df = reduce_mem_usage(test_df)
 gc.collect()<define_variables>
print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...' )