Kaggle competition: University of Liverpool - Ion Switching
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<import_modules>
pd.set_option('display.max_rows', 1000)
 train = pd.read_csv('/kaggle/input/ionswitchingkl/datasets/trainK.csv' )
import torch.nn.functional as F 
<define_search_model>
pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 1000)
 pd.set_option('display.max_colwidth', 199)
 warnings.filterwarnings("ignore" )
class Wave_Block(nn.Module):
def __init__(self,in_channels,out_channels,dilation_rates):
super(Wave_Block,self ).__init__()
self.num_rates = dilation_rates
self.convs = nn.ModuleList()
self.filter_convs = nn.ModuleList()
self.gate_convs = nn.ModuleList()
self.convs.append(nn.Conv1d(in_channels,out_channels,kernel_size=1))
dilation_rates = [2**i for i in range(dilation_rates)]
for dilation_rate in dilation_rates:
self.filter_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))
self.gate_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))
self.convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=1))
def forward(self,x):
x = self.convs[0](x)
 res = x
for i in range(self.num_rates):
x = F.tanh(self.filter_convs[i](x)) *F.sigmoid(self.gate_convs[i](x))
x = self.convs[i+1](x)
 res = torch.add(res, x)
 return res
class Classifier(nn.Module):
def __init__(self):
super().__init__()
input_size = 128
self.LSTM1 = nn.GRU(input_size=19,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)
 self.LSTM = nn.GRU(input_size=input_size,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)
 self.wave_block1 = Wave_Block(128,16,12)
 self.wave_block2 = Wave_Block(16,32,8)
 self.wave_block3 = Wave_Block(32,64,4)
 self.wave_block4 = Wave_Block(64, 128, 1)
 self.fc = nn.Linear(128, 11)
 def forward(self,x):
x,_ = self.LSTM1(x)
 x = x.permute(0, 2, 1)
 x = self.wave_block1(x)
 x = self.wave_block2(x)
 x = self.wave_block3(x)
 x = self.wave_block4(x)
 x = x.permute(0, 2, 1)
 x,_ = self.LSTM(x)
 x = self.fc(x)
 return x
class EarlyStopping:
def __init__(self, patience=7, delta=0, checkpoint_path='checkpoint.pt', is_maximize=True):
self.patience, self.delta, self.checkpoint_path = patience, delta, checkpoint_path
self.counter, self.best_score = 0, None
self.is_maximize = is_maximize
def load_best_weights(self, model):
model.load_state_dict(torch.load(self.checkpoint_path))
def __call__(self, score, model):
if self.best_score is None or \ 
(score > self.best_score + self.delta if self.is_maximize else score < self.best_score - self.delta):
torch.save(model.state_dict() , self.checkpoint_path)
 self.best_score, self.counter = score, 0
return 1
else:
self.counter += 1
if self.counter >= self.patience:
return 2
return 0<train_model>
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 print('Pre-Processing stat features.... ')
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['var'+c] = df.groupby([c])['signal'].var()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['range'+c] = df['max'+c] - df['min'+c]
df['maxtomin'+c] = df['max'+c] / df['min'+c]
df['abs_avg'+c] =(df['abs_min'+c] + df['abs_max'+c])/ 2
print('Pre-Processing shifting features.... ')
 df['shift+1_'] = df['signal'].shift(1)
 df['shift-1_'] = df['signal'].shift(-1)
 df['shift+2_'] = df['signal'].shift(2)
 df['shift-2_'] = df['signal'].shift(-2)
 df['shift+3_'] = df['signal'].shift(3)
 df['shift-3_'] = df['signal'].shift(-3)
 df['shift+4_'] = df['signal'].shift(4)
 df['shift-4_'] = df['signal'].shift(-4)
 df['shift+5_'] = df['signal'].shift(5)
 df['shift-5_'] = df['signal'].shift(-5)
 for c in [20,50,100,500]:
print('Pre-Processing rolling window '+str(c)+'.....')
 df['rolling_mean'+str(c)] = df['signal'].rolling(c ).mean().tolist()
df['rolling_std'+str(c)] = df['signal'].rolling(c ).std().tolist()
df['rolling_var'+str(c)] = df['signal'].rolling(c ).var().tolist()
df = df.fillna(method='bfill' ).fillna(method='ffill')
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]:
df[c+'_msignal'] = df[c] - df['signal']
return df
training = features(train )
class IronDataset(Dataset):
def __init__(self, data, labels, training=True, transform=None, seq_len=5000, flip=0.5, noise_level=0, class_split=0.0):
self.data = data
self.labels = labels
self.transform = transform
self.training = training
self.flip = flip
self.noise_level = noise_level
self.class_split = class_split
self.seq_len = seq_len
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
data = self.data[idx]
labels = self.labels[idx]
return [data.astype(np.float32), labels.astype(int)]<split>
del train
train, test, train_tr, new_splits = split()<install_modules>
training_columns = training.columns
col = [c for c in training_columns if c not in ['time', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]
Y = training['open_channels'].values
pip install torchcontrib<import_modules>
X = training[col].values
del training
from torchcontrib.optim import SWA
import torchcontrib<choose_model_class>
gc.collect()
model = Classifier()
model<create_dataframe>
X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size=0.3, random_state=7)
 del X
del Y
%%time
test_y = np.zeros([int(2000000/GROUP_BATCH_SIZE), GROUP_BATCH_SIZE, 1])
 test_dataset = IronDataset(test, test_y, flip=False)
 test_dataloader = DataLoader(test_dataset, NNBATCHSIZE, shuffle=False)
 test_preds_all = np.zeros(( 2000000, 11))
oof_score = []
for index,(train_index, val_index, _)in enumerate(new_splits[0:], start=0):
print("Fold : {}".format(index))
train_dataset = IronDataset(train[train_index], train_tr[train_index], seq_len=GROUP_BATCH_SIZE, flip=flip, noise_level=noise)
 train_dataloader = DataLoader(train_dataset, NNBATCHSIZE, shuffle=True,num_workers = 16)
 valid_dataset = IronDataset(train[val_index], train_tr[val_index], seq_len=GROUP_BATCH_SIZE, flip=False)
 valid_dataloader = DataLoader(valid_dataset, NNBATCHSIZE, shuffle=False)
 it = 0
model = Classifier()
model = model.cuda()
early_stopping = EarlyStopping(patience=40, is_maximize=True,
checkpoint_path=os.path.join(outdir, "gru_clean_checkpoint_fold_{}_iter_{}.pt".format(index,
it)))
 weight = None
criterion = nn.CrossEntropyLoss(weight=weight)
 optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 optimizer = torchcontrib.optim.SWA(optimizer, swa_start=10, swa_freq=2, swa_lr=0.0011)
 schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=2, factor=0.2)
 avg_train_losses, avg_valid_losses = [], []
for epoch in range(EPOCHS):
train_losses, valid_losses = [], []
tr_loss_cls_item, val_loss_cls_item = [], []
model.train()
train_preds, train_true = torch.Tensor([] ).cuda() , torch.LongTensor([] ).cuda()
print('**********************************')
 print("Folder : {} Epoch : {}".format(index, epoch))
print("Curr learning_rate: {:0.9f}".format(optimizer.param_groups[0]['lr']))
for x, y in tqdm(train_dataloader):
x = x.cuda()
y = y.cuda()
optimizer.zero_grad()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 y_ = y.view(-1)
 loss = criterion(predictions_, y_)
 loss.backward()
optimizer.step()
schedular.step(loss)
 train_losses.append(loss.item())
 train_true = torch.cat([train_true, y_], 0)
 train_preds = torch.cat([train_preds, predictions_], 0)
 optimizer.update_swa()
optimizer.swap_swa_sgd()
val_preds, val_true = torch.Tensor([] ).cuda() , torch.LongTensor([] ).cuda()
print('EVALUATION')
 with torch.no_grad() :
for x, y in tqdm(valid_dataloader):
x = x.cuda()
y = y.cuda()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 y_ = y.view(-1)
 loss = criterion(predictions_, y_)
 valid_losses.append(loss.item())
 val_true = torch.cat([val_true, y_], 0)
 val_preds = torch.cat([val_preds, predictions_], 0)
 train_loss = np.average(train_losses)
 valid_loss = np.average(valid_losses)
 avg_train_losses.append(train_loss)
 avg_valid_losses.append(valid_loss)
 print("train_loss: {:0.6f}, valid_loss: {:0.6f}".format(train_loss, valid_loss))
train_score = f1_score(train_true.cpu().detach().numpy() , train_preds.cpu().detach().numpy().argmax(1),
labels=list(range(11)) , average='macro')
 val_score = f1_score(val_true.cpu().detach().numpy() , val_preds.cpu().detach().numpy().argmax(1),
labels=list(range(11)) , average='macro')
 schedular.step(val_score)
 print("train_f1: {:0.6f}, valid_f1: {:0.6f}".format(train_score, val_score))
res = early_stopping(val_score, model)
 if res == 2:
print("Early Stopping")
 print('folder %d global best val max f1 model score %f' %(index, early_stopping.best_score))
break
elif res == 1:
print('save folder %d global val max f1 model score %f' %(index, val_score))
print('Folder {} finally best global max f1 score is {}'.format(index, early_stopping.best_score))
oof_score.append(round(early_stopping.best_score, 6))
model.eval()
pred_list = []
with torch.no_grad() :
for x, y in tqdm(test_dataloader):
x = x.cuda()
y = y.cuda()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 pred_list.append(F.softmax(predictions_, dim=1 ).cpu().numpy())
 test_preds = np.vstack(pred_list)
 test_preds_all += test_preds 
<save_to_csv>
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True)
 params = {}
params['application']='root_mean_squared_error'
params['num_boost_round'] = 3000
params['learning_rate'] = 0.01
params['boosting_type'] = 'gbdt'
params['metric'] = 'rmse'
params['sub_feature'] = 0.833
params['num_leaves'] = 207
params['min_split_gain'] = 0.05
params['min_child_weight'] = 27
params['max_depth'] = -1
params['num_threads'] = 20
params['max_bin'] = 50
params['lambda_l2'] = 0.10
params['lambda_l1'] = 0.30
params['feature_fraction']= 0.833
params['bagging_fraction']= 0.979
params['seed']=1729
params['device_type'] = 'gpu'
model = lgb.train(params, lgb.Dataset(X_train, Y_train), 2000, lgb.Dataset(X_test, Y_test),verbose_eval=100, early_stopping_rounds=200, feval=MacroF1Metric )
print('all folder score is:%s'%str(oof_score))
print('OOF mean score is: %f'%(sum(oof_score)/len(oof_score)))
 print('Generate submission.............')
 submission_csv_path = '/kaggle/input/liverpool-ion-switching/sample_submission.csv'
ss = pd.read_csv(submission_csv_path, dtype={'time': str})
 test_preds_all = test_preds_all / np.sum(test_preds_all, axis=1)[:, None]
test_pred_frame = pd.DataFrame({'time': ss['time'].astype(str),
'open_channels': np.argmax(test_preds_all, axis=1)})
 test_pred_frame.to_csv("./gru_preds.csv", index=False)
 print('over' )<load_from_csv>
del X_train
del X_test
del Y_train
del Y_test
submit = pd.read_csv("/kaggle/input/liverpool-ion-switching/sample_submission.csv")
 paths = {
"wavenet": "/kaggle/input/wavenet-keras/submission.csv",
"kalman": "/kaggle/input/single-model-lgbm-kalman-filter/submission.csv",
"rnn": "/kaggle/input/seq2seq-rnn-with-gru/gru_preds.csv",
"lgb-fe": "/kaggle/input/lgb-featureengineering-lb-0-940/submission.csv",
"lstm": "/kaggle/input/simple-two-layer-bidirectional-lstm-with-pytorch/submission_bilstm.csv",
}
weights = np.array([0.941, 0.94, 0.94, 0.94, 0.94])
 subs = submit.copy()
for i, p in enumerate(paths.keys()):
tmp = pd.read_csv(paths[p])
 subs[f"{p}"] = tmp["open_channels"]
subs.drop(columns=["time", "open_channels"], inplace=True )<statistical_test>
gc.collect()
def weighted_voting(row):
h = np.histogram(row.values, weights=weights)
 return np.round(h[1][np.argmax(h[0])] )<feature_engineering>
test = pd.read_csv('/kaggle/input/ionswitchingkl/datasets/testK.csv')
 test = features(test )
submit["open_channels"] = subs.apply(weighted_voting, axis=1 )<save_to_csv>
preds = model.predict(test[col], num_iteration=model.best_iteration)
 test['open_channels'] = np.round(np.clip(preds, 0, 10)).astype(int)
 test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )