Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<feature_engineering>
import numpy as np
import pandas as pd
import os
submission.selected_text = prediction1<save_to_csv>
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv' )
submission.to_csv("submission.csv",index=False )<install_modules>
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
!pip install transformers<import_modules>
EPOCHS = 150
NNBATCHSIZE = 32
GROUP_BATCH_SIZE = 4000
SEED = 123
LR = 0.001
SPLITS = 5
outdir = 'wavenet_models'
flip = False
noise = False
if not os.path.exists(outdir):
os.makedirs(outdir)
 def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
import tensorflow as tf
import keras
import transformers
from transformers import *
import pandas as pd
import numpy as np
import tokenizers<import_modules>
def read_data() :
train = pd.read_csv('/kaggle/input/clean-kalman/train_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/clean-kalman/test_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def split(GROUP_BATCH_SIZE=4000, SPLITS=5):
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size=GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size=GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print(train.head())
 print('Feature Engineering Completed...')
 target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 target_cols = ['open_channels']
print(train.head() , train.shape)
 train_tr = np.array(list(train.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[features].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[features].values)))
 print(train.shape, test.shape, train_tr.shape)
 return train, test, train_tr, new_splits
print(tf.__version__ )<string_transform>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
vocab_file = '.. /input/tf-roberta/vocab-roberta-base.json'
merge_file = '.. /input/tf-roberta/merges-roberta-base.txt'
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file, merge_file,lowercase = True )<load_from_csv>
import torch.nn.functional as F
MAX_LEN = 100
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
class Wave_Block(nn.Module):
def __init__(self,in_channels,out_channels,dilation_rates):
super(Wave_Block,self ).__init__()
self.num_rates = dilation_rates
self.convs = nn.ModuleList()
self.filter_convs = nn.ModuleList()
self.gate_convs = nn.ModuleList()
self.convs.append(nn.Conv1d(in_channels,out_channels,kernel_size=1))
dilation_rates = [2**i for i in range(dilation_rates)]
for dilation_rate in dilation_rates:
self.filter_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))
self.gate_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))
self.convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=1))
def forward(self,x):
x = self.convs[0](x)
 res = x
for i in range(self.num_rates):
x = F.tanh(self.filter_convs[i](x)) *F.sigmoid(self.gate_convs[i](x))
x = self.convs[i+1](x)
 res = torch.add(res, x)
 return res
class Classifier(nn.Module):
def __init__(self):
super().__init__()
input_size = 128
self.LSTM1 = nn.GRU(input_size=19,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)
 self.LSTM = nn.GRU(input_size=input_size,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)
 self.wave_block1 = Wave_Block(128,16,12)
 self.wave_block2 = Wave_Block(16,32,8)
 self.wave_block3 = Wave_Block(32,64,4)
 self.wave_block4 = Wave_Block(64, 128, 1)
 self.fc = nn.Linear(128, 11)
 def forward(self,x):
x,_ = self.LSTM1(x)
 x = x.permute(0, 2, 1)
 x = self.wave_block1(x)
 x = self.wave_block2(x)
 x = self.wave_block3(x)
 x = self.wave_block4(x)
 x = x.permute(0, 2, 1)
 x,_ = self.LSTM(x)
 x = self.fc(x)
 return x
class EarlyStopping:
def __init__(self, patience=7, delta=0, checkpoint_path='checkpoint.pt', is_maximize=True):
self.patience, self.delta, self.checkpoint_path = patience, delta, checkpoint_path
self.counter, self.best_score = 0, None
self.is_maximize = is_maximize
def load_best_weights(self, model):
model.load_state_dict(torch.load(self.checkpoint_path))
def __call__(self, score, model):
if self.best_score is None or \ 
(score > self.best_score + self.delta if self.is_maximize else score < self.best_score - self.delta):
torch.save(model.state_dict() , self.checkpoint_path)
 self.best_score, self.counter = score, 0
return 1
else:
self.counter += 1
if self.counter >= self.patience:
return 2
return 0
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
class IronDataset(Dataset):
def __init__(self, data, labels, training=True, transform=None, seq_len=5000, flip=0.5, noise_level=0, class_split=0.0):
self.data = data
self.labels = labels
self.transform = transform
self.training = training
self.flip = flip
self.noise_level = noise_level
self.class_split = class_split
self.seq_len = seq_len
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
data = self.data[idx]
labels = self.labels[idx]
return [data.astype(np.float32), labels.astype(int)]
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
train, test, train_tr, new_splits = split()
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained('.. /input/tf-roberta/config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained('.. /input/tf-roberta/pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 drop1 = tf.keras.layers.Dropout(0.2 )(x[0])
 layer2 = tf.keras.layers.Conv1D(50 ,kernel_size = 1 )(drop1)
 drop2 = tf.keras.layers.Dropout(0.2 )(layer2)
 layer3 = tf.keras.layers.Conv1D(1 ,kernel_size = 1 )(drop2)
 layer4 = tf.keras.layers.Flatten()(layer3)
 output_1 = tf.keras.layers.Activation('softmax' )(layer4)
 drop1_ = tf.keras.layers.Dropout(0.2 )(x[0])
 layer1_ = tf.keras.layers.Conv1D(50 ,kernel_size = 1 )(drop1_)
 drop2_ = tf.keras.layers.Dropout(0.2 )(layer1_)
 layer2_ = tf.keras.layers.Conv1D(1 ,kernel_size = 1 )(drop2_)
 layer3_ = tf.keras.layers.Flatten()(layer2_)
 output_2 = tf.keras.layers.Activation('softmax' )(layer3_)
 model = tf.keras.Model(inputs = [ids ,att ,tok] ,outputs = [output_1 ,output_2])
 model.summary()<compute_test_metric>
pip install torchcontrib
def my_loss(alpha ,gamma):

 def focal_loss(y_true ,y_pred):
y_true = tf.cast(y_true ,dtype = tf.float32)
 y_pred = tf.cast(y_pred ,dtype = tf.float32)
 log_lik = y_true*tf.keras.backend.log(y_pred)
 log_lik = alpha*(( 1-y_pred)**gamma)*log_lik
return -tf.keras.backend.sum(log_lik)
 return focal_loss<choose_model_class>
from torchcontrib.optim import SWA
import torchcontrib
optimizer = tf.keras.optimizers.Adam(learning_rate=0.000001)
 model.compile(loss = my_loss(1.0 ,2.0),optimizer = optimizer )<train_model>
model = Classifier()
model
history = model.fit([input_ids[800:], attention_mask[800:],token_type_ids[800:]], [start_tokens[800:], end_tokens[800:]] ,epochs = 30 ,
validation_split = 0.1 ,batch_size = 32 )<string_transform>
%%time
test_y = np.zeros([int(2000000/GROUP_BATCH_SIZE), GROUP_BATCH_SIZE, 1])
 test_dataset = IronDataset(test, test_y, flip=False)
 test_dataloader = DataLoader(test_dataset, NNBATCHSIZE, shuffle=False)
 test_preds_all = np.zeros(( 2000000, 11))
oof_score = []
for index,(train_index, val_index, _)in enumerate(new_splits[0:], start=0):
print("Fold : {}".format(index))
train_dataset = IronDataset(train[train_index], train_tr[train_index], seq_len=GROUP_BATCH_SIZE, flip=flip, noise_level=noise)
 train_dataloader = DataLoader(train_dataset, NNBATCHSIZE, shuffle=True,num_workers = 16)
 valid_dataset = IronDataset(train[val_index], train_tr[val_index], seq_len=GROUP_BATCH_SIZE, flip=False)
 valid_dataloader = DataLoader(valid_dataset, NNBATCHSIZE, shuffle=False)
 it = 0
model = Classifier()
model = model.cuda()
early_stopping = EarlyStopping(patience=40, is_maximize=True,
checkpoint_path=os.path.join(outdir, "gru_clean_checkpoint_fold_{}_iter_{}.pt".format(index,
it)))
 weight = None
criterion = nn.CrossEntropyLoss(weight=weight)
 optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 optimizer = torchcontrib.optim.SWA(optimizer, swa_start=10, swa_freq=2, swa_lr=0.0011)
 schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=2, factor=0.2)
 avg_train_losses, avg_valid_losses = [], []
for epoch in range(EPOCHS):
train_losses, valid_losses = [], []
tr_loss_cls_item, val_loss_cls_item = [], []
model.train()
train_preds, train_true = torch.Tensor([] ).cuda() , torch.LongTensor([] ).cuda()
print('**********************************')
 print("Folder : {} Epoch : {}".format(index, epoch))
print("Curr learning_rate: {:0.9f}".format(optimizer.param_groups[0]['lr']))
for x, y in tqdm(train_dataloader):
x = x.cuda()
y = y.cuda()
optimizer.zero_grad()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 y_ = y.view(-1)
 loss = criterion(predictions_, y_)
 loss.backward()
optimizer.step()
schedular.step(loss)
 train_losses.append(loss.item())
 train_true = torch.cat([train_true, y_], 0)
 train_preds = torch.cat([train_preds, predictions_], 0)
 optimizer.update_swa()
optimizer.swap_swa_sgd()
val_preds, val_true = torch.Tensor([] ).cuda() , torch.LongTensor([] ).cuda()
print('EVALUATION')
 with torch.no_grad() :
for x, y in tqdm(valid_dataloader):
x = x.cuda()
y = y.cuda()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 y_ = y.view(-1)
 loss = criterion(predictions_, y_)
 valid_losses.append(loss.item())
 val_true = torch.cat([val_true, y_], 0)
 val_preds = torch.cat([val_preds, predictions_], 0)
 train_loss = np.average(train_losses)
 valid_loss = np.average(valid_losses)
 avg_train_losses.append(train_loss)
 avg_valid_losses.append(valid_loss)
 print("train_loss: {:0.6f}, valid_loss: {:0.6f}".format(train_loss, valid_loss))
train_score = f1_score(train_true.cpu().detach().numpy() , train_preds.cpu().detach().numpy().argmax(1),
labels=list(range(11)) , average='macro')
 val_score = f1_score(val_true.cpu().detach().numpy() , val_preds.cpu().detach().numpy().argmax(1),
labels=list(range(11)) , average='macro')
 schedular.step(val_score)
 print("train_f1: {:0.6f}, valid_f1: {:0.6f}".format(train_score, val_score))
res = early_stopping(val_score, model)
 if res == 2:
print("Early Stopping")
 print('folder %d global best val max f1 model score %f' %(index, early_stopping.best_score))
break
elif res == 1:
print('save folder %d global val max f1 model score %f' %(index, val_score))
print('Folder {} finally best global max f1 score is {}'.format(index, early_stopping.best_score))
oof_score.append(round(early_stopping.best_score, 6))
model.eval()
pred_list = []
with torch.no_grad() :
for x, y in tqdm(test_dataloader):
x = x.cuda()
y = y.cuda()
predictions = model(x)
 predictions_ = predictions.view(-1, predictions.shape[-1])
 pred_list.append(F.softmax(predictions_, dim=1 ).cpu().numpy())
 test_preds = np.vstack(pred_list)
 test_preds_all += test_preds
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<predict_on_test>
print('all folder score is:%s'%str(oof_score))
print('OOF mean score is: %f'%(sum(oof_score)/len(oof_score)))
 print('Generate submission.............')
 submission_csv_path = '/kaggle/input/liverpool-ion-switching/sample_submission.csv'
ss = pd.read_csv(submission_csv_path, dtype={'time': str})
 test_preds_all = test_preds_all / np.sum(test_preds_all, axis=1)[:, None]
test_pred_frame = pd.DataFrame({'time': ss['time'].astype(str),
'open_channels': np.argmax(test_preds_all, axis=1)})
 test_pred_frame.to_csv("./gru_preds.csv", index=False)
 print('over' )