Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<save_to_csv>
device = torch.device("cuda:0")if torch.cuda.is_available() else torch.device("cpu")
test['target'] = ensemble_preds
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission.csv' , index=False)
 output.head()<save_to_csv>
n_epochs = 250
lr = 0.01
n_folds = 5
lstm_input_size = 1
hidden_state_size = 30
batch_size = 30
num_sequence_layers = 2
output_dim = 11
num_time_steps = 4000
rnn_type = 'LSTM'
test['target'] = cb_test_preds
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_1.csv' , index=False)
 output.head()<predict_on_test>
class Bi_RNN(nn.Module):
def __init__(self, input_dim, hidden_dim, batch_size, output_dim=11, num_layers=2, rnn_type='LSTM'):
super(Bi_RNN, self ).__init__()
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.batch_size = batch_size
self.num_layers = num_layers
self.init_linear = nn.Linear(self.input_dim, self.input_dim)
 self.lstm = eval('nn.' + rnn_type )(self.input_dim, self.hidden_dim, self.num_layers, batch_first=True, bidirectional=True)
 self.linear = nn.Linear(self.hidden_dim * 2, output_dim)
 def init_hidden(self):
return(torch.zeros(self.num_layers, self.batch_size, self.hidden_dim),
torch.zeros(self.num_layers, self.batch_size, self.hidden_dim))
def forward(self, input):
linear_input = self.init_linear(input)
 lstm_out, self.hidden = self.lstm(linear_input)
 y_pred = self.linear(lstm_out)
 return y_pred
X = ensemble_df
y = y_val_xgb
dregr = DecisionTreeRegressor(max_depth=5)
 dregr.fit(X, y)
 ensemble_preds_dr = dregr.predict(ensemble_test_df)
<save_to_csv>
class ION_Dataset_Sequential(Dataset):
def __init__(self, input, output):
self.input = input
self.output = output
def __len__(self):
return len(self.input)
 def __getitem__(self, idx):
x = self.input[idx]
y = self.output[idx]
x = torch.tensor(x, dtype=torch.float)
 y = torch.tensor(y, dtype=torch.float)
 return x, y
class ION_Dataset_Sequential_test(Dataset):
def __init__(self, input):
self.input = input
def __len__(self):
return len(self.input)
 def __getitem__(self, idx):
x = self.input[idx]
x = torch.tensor(x, dtype=torch.float)
 return x
test['target'] = ensemble_preds_dr
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_2.csv' , index=False)
 output.head()<train_model>
train_df = pd.read_csv('/kaggle/input/data-no-drift/train_detrend.csv')
 test_df = pd.read_csv('/kaggle/input/data-no-drift/test_detrend.csv')
 X = train_df['signal'].values.reshape(-1, num_time_steps, 1)
 y = pd.get_dummies(train_df['open_channels'] ).values.reshape(-1, num_time_steps, output_dim)
 test_input = test_df["signal"].values.reshape(-1, num_time_steps, 1)
 train_input_mean = X.mean()
train_input_sigma = X.std()
test_input =(test_input-train_input_mean)/train_input_sigma
test_preds = np.zeros(( int(test_input.shape[0] * test_input.shape[1])))
 test = ION_Dataset_Sequential_test(test_input)
 test_loader = DataLoader(test, batch_size=batch_size, shuffle=False )
X = ensemble_df
y = y_val_xgb
rfregr = RandomForestRegressor(n_estimators = 200 , max_depth=5 , min_samples_leaf=100 , n_jobs=4)
 rfregr.fit(X, y)
 ensemble_preds_rf = rfregr.predict(ensemble_test_df )<save_to_csv>
kfold = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 local_val_score = 0
models = {}
k=0
for tr_idx, val_idx in kfold.split(X, y):
test_p = np.zeros(( int(test_input.shape[0] * test_input.shape[1])))
 print('starting fold', k)
 k += 1
print(6*'
train_input = X[tr_idx]
print(train_input.shape)
 train_target = y[tr_idx]
val_input = X[val_idx]
val_target = y[val_idx]
train_input_mean = train_input.mean()
train_input_sigma = train_input.std()
val_input =(val_input-train_input_mean)/train_input_sigma
train_input =(train_input-train_input_mean)/train_input_sigma
print(6*'
train = ION_Dataset_Sequential(train_input, train_target)
 valid = ION_Dataset_Sequential(val_input, val_target)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 train_preds = np.zeros(( int(train_input.shape[0] * train_input.shape[1])))
 val_preds = np.zeros(( int(val_input.shape[0] * val_input.shape[1])))
 best_val_preds = np.zeros(( int(val_input.shape[0] * val_input.shape[1])))
 train_targets = np.zeros(( int(train_input.shape[0] * train_input.shape[1])))
 avg_losses_f = []
avg_val_losses_f = []
loss_fn = torch.nn.BCEWithLogitsLoss()
model = Bi_RNN(lstm_input_size, hidden_state_size, batch_size=batch_size, output_dim=output_dim, num_layers=num_sequence_layers, rnn_type=rnn_type)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=1e-5)
 scheduler = ReduceLROnPlateau(optimizer, 'min', patience=150, factor=0.1, min_lr=1e-8)
 temp_val_loss = 9999999999
reached_val_score = 0
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch)in enumerate(train_loader):
x_batch = x_batch.view(-1, num_time_steps, lstm_input_size)
 y_batch = y_batch.view(-1, num_time_steps, output_dim)
 optimizer.zero_grad()
y_pred = model(x_batch.cuda())
 loss = loss_fn(y_pred.cpu() , y_batch)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 pred = F.softmax(y_pred, 2 ).detach().cpu().numpy().argmax(axis=-1)
 train_preds[i * batch_size * train_input.shape[1]:(i + 1)* batch_size * train_input.shape[1]] = pred.reshape(( -1))
train_targets[i * batch_size * train_input.shape[1]:(i + 1)* batch_size * train_input.shape[1]] = y_batch.detach().cpu().numpy().argmax(axis=2 ).reshape(( -1))
del y_pred, loss, x_batch, y_batch, pred
model.eval()
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
x_batch = x_batch.view(-1, num_time_steps, lstm_input_size)
 y_batch = y_batch.view(-1, num_time_steps, output_dim)
 y_pred = model(x_batch.cuda() ).detach()
avg_val_loss += loss_fn(y_pred.cpu() , y_batch ).item() / len(valid_loader)
 pred = F.softmax(y_pred, 2 ).detach().cpu().numpy().argmax(axis=-1)
 val_preds[i * batch_size * val_input.shape[1]:(i + 1)* batch_size * val_input.shape[1]] = pred.reshape(( -1))
del y_pred, x_batch, y_batch, pred
if avg_val_loss < temp_val_loss:
temp_val_loss = avg_val_loss
train_score = f1_score(train_targets, train_preds, average='macro')
 val_score = f1_score(val_target.argmax(axis=2 ).reshape(( -1)) , val_preds, average='macro')
 elapsed_time = time.time() - start_time
scheduler.step(avg_val_loss)
 if epoch%10 == 0:
print('Epoch {}/{} \t loss={:.4f} \t train_f1={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} \t time={:.2f}s'.format(epoch + 1, n_epochs, avg_loss, train_score, avg_val_loss, val_score, elapsed_time))
if val_score > reached_val_score:
reached_val_score = val_score
best_model = copy.deepcopy(model.state_dict())
 best_val_preds = copy.deepcopy(val_preds)
 val_score_fold = f1_score(val_target.argmax(axis=2 ).reshape(( -1)) , best_val_preds, average='macro')
 models[k] = best_model
print("BEST VALIDATION SCORE(F1): ", val_score_fold)
 local_val_score +=(1/n_folds)* val_score_fold
print("Final Score ", local_val_score )
test['target'] = ensemble_preds_rf
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_3.csv' , index=False)
 output.head()<define_variables>
for k in range(n_folds):
test_p = np.zeros(( int(test_input.shape[0] * test_input.shape[1])))
 k += 1
model = Bi_RNN(lstm_input_size, hidden_state_size, batch_size=batch_size, output_dim=output_dim, num_layers=num_sequence_layers, rnn_type=rnn_type)
 model = model.to(device)
 model.load_state_dict(models[k])
 model.eval()
for i, x_batch in enumerate(test_loader):
x_batch = x_batch.view(-1, num_time_steps, lstm_input_size)
 y_pred = model(x_batch.cuda() ).detach()
pred = F.softmax(y_pred, 2 ).detach().cpu().numpy().argmax(axis=-1)
 test_p[i * batch_size * test_input.shape[1]:(i + 1)* batch_size * test_input.shape[1]] = pred.reshape(( -1))
del y_pred, x_batch, pred
test_preds +=(1/n_folds)* test_p
<categorify><EOS> 
df_sub = pd.read_csv("/kaggle/input/liverpool-ion-switching/sample_submission.csv", dtype = {'time': str})
 df_sub.open_channels = np.array(test_preds, np.int)
 df_sub.to_csv("submission_bilstm.csv", index=False )