Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<load_from_csv>
sns.set_style("darkgrid")
 warnings.filterwarnings("ignore" )
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 base_path = '.. /input/clean-kalman/'
base_path0 = '.. /input/liverpool-ion-switching'
train_path = os.path.join(base_path, 'train_clean_kalman.csv')
 test_path = os.path.join(base_path, 'test_clean_kalman.csv')
 sub_path = os.path.join(base_path0, 'sample_submission.csv')
 fold_split = 10
window_list = [20, 50]
SAMPLE = 10000
FOLD_GET = 5
RANDOM_NOISE = 0.001
kf = StratifiedKFold(n_splits=fold_split, random_state=37, shuffle=True )
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(config.ROBERTA_PATH + "config.json")
 model_config.output_hidden_states = True<find_best_params>
train_df = pd.read_csv(train_path)
 sns.distplot(train_df['signal'].values)
 plt.show()
train_df.tail()
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load("model_0.pth"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load("model_1.pth"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load("model_2.pth"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load("model_3.pth"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load("model_4.pth"))
model5.eval()<create_dataframe>
%%time
def Kalman1D(observations,damping=1):
observation_covariance = damping
initial_value_guess = observations[0]
transition_matrix = 1
transition_covariance = 0.1
initial_value_guess
kf = KalmanFilter(
 initial_state_mean=initial_value_guess,
initial_state_covariance=observation_covariance,
observation_covariance=observation_covariance,
transition_covariance=transition_covariance,
transition_matrices=transition_matrix 
)
 pred_state, state_cov = kf.smooth(observations)
 return pred_state
observation_covariance = 0.0015
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=1,
collate_fn = collate_fn 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(
 outputs_start1
+ outputs_start2
+ outputs_start3
+ outputs_start4
+ outputs_start5 
)/ 5
outputs_end =(
 outputs_end1
+ outputs_end2
+ outputs_end3
+ outputs_end4
+ outputs_end5 
)/ 5
outputs_start = outputs_start
outputs_end = outputs_end
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.where(outputs_start[px, :].cumsum(axis = 0)> cdf_threshold)[0].min() ,
idx_end=np.where(outputs_end[px, :][::-1].cumsum(axis = 0)[::-1] > cdf_threshold)[0].max() ,
offsets=offsets[px] 
)
 final_output.append(output_sentence )<save_to_csv>
def normalize(train, test, col, type_='StandardScaler'):
if type_ == 'StandardScaler':
clf = StandardScaler()
elif type_ == 'MinMaxScaler':
clf = MinMaxScaler()
elif type_ == 'RobustScaler':
clf = RobustScaler()
else:
print('Error type in!')
 train[col] = clf.fit_transform(train[col])
 test[col] = clf.transform(test[col])
 return train, test
train_df, test_df = normalize(train_df, test_df, ['signal'])
 gc.collect()
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.to_csv("submission.csv", index=False )<import_modules>
channel_signal = train_df.groupby('open_channels')['signal'].agg(['min', 'max', np.mean, np.std] ).reset_index()
channel_signal.columns = ['signal_' + i if i!='open_channels' else i for i in channel_signal.columns]
print(channel_signal)
 try:
del channel_signal
except:
print('Variable channel_signal not defined!' )
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense,Input,Embedding
from tensorflow.keras.models import Model
from tensorflow.keras.models import load_model
import keras.backend as K
from transformers import *
import tokenizers
from tokenizers import ByteLevelBPETokenizer
import os
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder<load_from_csv>
def process_data(df, windows=window_list, batch_size=20000):
df = df.sort_values('time' ).reset_index(drop=True)
 df.index =(( df.time*10000)- 1 ).values
df['batch'] = df.index // batch_size
df['batch_index'] = df.index % batch_size
df['batch_slices'] = df['batch_index'] //(batch_size//10)
 df['batch_slices1'] = df.apply(lambda x: '_'.join([str(x['batch'] ).zfill(3), str(x['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch', 'batch_slices1']:
d = {}
d[f'min_{c}'] = df.groupby(c)['signal'].min()
d[f'max_{c}'] = df.groupby(c)['signal'].max()
d[f'mean_{c}'] = df.groupby(c)['signal'].mean()
d[f'std_{c}'] = df.groupby(c)['signal'].std()
for per in [10, 25, 75, 90]:
d[f'pct_{per}_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.percentile(x, per))
d[f'skew_{c}'] = df.groupby(c)['signal'].apply(lambda x: pd.Series(x ).skew())
 d[f'kurtosis_{c}'] = df.groupby(c)['signal'].apply(lambda x: pd.Series(x ).kurtosis())
 d[f'mean_abs_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d[f'min_abs_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.min(np.abs(x)))
 d[f'max_abs_{c}'] = df.groupby(c)['signal'].apply(lambda x: np.max(np.abs(x)))
 d[f'range_{c}'] = d[f'max_{c}'] - d[f'min_{c}']
d[f'ratio_{c}'] = d[f'max_{c}'] / d[f'min_{c}']
d[f'avg_abs_{c}'] =(d[f'min_abs_{c}'] + d[f'max_abs_{c}'])/2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['shift_+1'] = [0] + list(df['signal'].values[:-1])
 df['shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['shift_+1'][i] = np.nan
for i in df[df['batch_index']==(batch_size-1)].index:
df['shift_-1'][i] = np.nan
df['shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['shift_-2'] = list(df['signal'].values[:-2])+ [0,] + [1,]
for i in df[df['batch_index'].isin([0, 1])].index:
df['shift_+2'][i] = np.nan
for i in df[df['batch_index'].isin([batch_size-1, batch_size-2])].index:
df['shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices1'])
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels']]:
df[f'{c}_msignal'] = df[f'{c}'] - df['signal']
df = df.replace([np.Inf, -np.Inf], np.nan)
 df.fillna(0, inplace=True)
 gc.collect()
return df
train_df = process_data(train_df)
 test_df = process_data(test_df )
train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv', keep_default_na=False)
 test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv', keep_default_na=False )<define_variables>
%%time
train_df = reduce_mem_usage(train_df)
 test_df = reduce_mem_usage(test_df)
 gc.collect()
Max_length = 120<categorify>
column_train = [i for i in train_df.columns if i not in ['time', 'signal', 'open_channels']]
column_train
def list_in(a, b):
y_seq = np.zeros(( len(b)) ,dtype='int32')
 for i in range(0,len(b)- len(a)+ 1):
if b[i:i + len(a)] == a:
y_seq[i:i+len(a)] = 1
if b[i-1] == " ":
y_seq[i-1] = 1
break
return y_seq
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
def custom_asymmetric_train(y_pred, y_true):
y_true = y_true.get_label()
residual =(y_true - y_pred ).astype('float')
 grad = np.where(residual < 0, -2*1.1*residual, -2*residual)
 hess = np.where(residual < 0, 2*1.1, 2)
 return grad, hess
def custom_asymmetric_valid(y_pred, y_true):
y_true = y_true.get_label()
residual =(y_true - y_pred ).astype('float')
 loss = np.where(residual < 0, residual**2, 1.15*(residual**2))
return 'custom_asymmetric_eval', np.mean(loss), False
def eval_gini(y_true, y_prob):
return np.sqrt(((y_prob - y_true)** 2 ).mean() )
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = "/kaggle/input/roberta-base/vocab-roberta-base.json" ,
merges_file = "/kaggle/input/roberta-base/merges-roberta-base.txt",
lowercase=True,
add_prefix_space=True )<categorify>
def model_train(model_type, params, train_df=train_df, test_df=test_df):
predict = np.zeros([len(test_df)])
 importances = np.zeros([len(column_train), FOLD_GET])
 for idx,(train_index, val_index)in enumerate(kf.split(train_df, train_df['open_channels'])) :
if idx % 2 == 0:
print(f'Fold_{idx}:')
 train_ = train_df.iloc[train_index]
val_ = train_df.iloc[val_index]
train_.reset_index(inplace=True, drop=True)
 val_.reset_index(inplace=True, drop=True)
 if model_type=='xgb':
train_dataset = lgb.DMatrix(train_.loc[:, column_train].values, label=train_.loc[:, 'open_channels'].values, feature_name=column_train)
 val_dataset = lgb.DMatrix(val_.loc[:, column_train].values, label=val_.loc[:, 'open_channels'].values, feature_name=column_train)
 clf = xgb.train(params, train_dataset, 500, valid_sets=[(val_dataset, 'valid')],
verbose_eval=100, early_stopping_rounds = 100)
 predict += clf.predict(test_df.loc[:, column_train], ntree_limit=clf.best_ntree_limit)/FOLD_GET
lgb.plot_importance(clf, max_num_features=10)
 elif model_type=='lgb':
train_dataset = lgb.Dataset(train_.loc[:, column_train].values, label=train_.loc[:, 'open_channels'].values, feature_name=column_train)
 val_dataset = lgb.Dataset(val_.loc[:, column_train].values, label=val_.loc[:, 'open_channels'].values, feature_name=column_train)
 clf = lgb.train(params, train_dataset, 4000, valid_sets=[train_dataset, val_dataset], feval=custom_asymmetric_valid,
fobj=custom_asymmetric_train, verbose_eval=500, early_stopping_rounds = 400)
 predict += clf.predict(test_df.loc[:, column_train], ntree_limit=clf.best_iteration)/FOLD_GET
del clf
gc.collect()
return predict, importances
Max_length = 120
obs = train.shape[0]
input_tokens = np.ones(( obs,Max_length),dtype='int32')
 mask = np.zeros(( obs,Max_length),dtype='int32')
 seg = np.zeros(( obs,Max_length),dtype='int32')
 st_vec = np.zeros(( obs,Max_length),dtype='int32')
 end_vec = np.zeros(( obs,Max_length),dtype='int32')
 for i in range(0,obs):
overlap_char = list_in(" ".join(train.selected_text.values[i].split())," "+" ".join(train.text.values[i].split()))
tweet_ex = tokenizer.encode(" "+" ".join(train.text.values[i].split()))
tweet_ex_ids = tweet_ex.ids
input_tokens[i,0:len(tweet_ex_ids)+5] = [0]+tweet_ex_ids+[2,2]+tokenizer.encode(train.sentiment.values[i] ).ids + [2]
mask[i,0:len(tweet_ex_ids)+5] = 1
tweet_ex_off = tweet_ex.offsets
selected_text_tweet = tokenizer.encode(" "+" ".join(train.selected_text.values[i].split()))
tokens = []
for j,(a,b)in enumerate(tweet_ex_off):
if sum(overlap_char[a:b])> 0:
tokens.append(j)
 if len(tokens)>0:
st_vec[i,tokens[0]+1] = 1
end_vec[i,tokens[-1]+1] = 1<categorify>
train_index, val_index = list(kf.split(train_df, train_df['open_channels'])) [0]
train_ = train_df.iloc[train_index]
val_ = train_df.iloc[val_index]
train_.reset_index(inplace=True, drop=True)
 val_.reset_index(inplace=True, drop=True)
 gc.collect()
obs1 = test.shape[0]
input_tokens_test = np.ones(( obs1,Max_length),dtype='int32')
 mask_test = np.zeros(( obs1,Max_length),dtype='int32')
 seg_test = np.zeros(( obs1,Max_length),dtype='int32')
 for i in range(0,obs1):
tweet_ex_test = tokenizer.encode(" "+" ".join(test.text.values[i].split()))
tweet_ex_ids_test = tweet_ex_test.ids
input_tokens_test[i,0:len(tweet_ex_ids_test)+5] = [0]+tweet_ex_ids_test+[2,2]+tokenizer.encode(test.sentiment.values[i] ).ids + [2]
mask_test[i,0:len(tweet_ex_ids_test)+5] = 1<load_pretrained>
%%time
def lgb_bayesian(
 num_leaves,
min_data_in_leaf,
learning_rate,
min_sum_hessian_in_leaf,
feature_fraction,
lambda_l1,
lambda_l2,
min_gain_to_split,
max_depth,
scale_pos_weight):
num_leaves = int(num_leaves)
 min_data_in_leaf = int(min_data_in_leaf)
 max_depth = int(max_depth)
 scale_pos_weight = int(scale_pos_weight)
 assert type(num_leaves)== int
assert type(min_data_in_leaf)== int
assert type(max_depth)== int
assert type(scale_pos_weight)== int
param = {
'num_leaves': num_leaves,
'max_bin': 63,
'min_data_in_leaf': min_data_in_leaf,
'learning_rate': learning_rate,
'min_sum_hessian_in_leaf': min_sum_hessian_in_leaf,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': feature_fraction,
'lambda_l1': lambda_l1,
'lambda_l2': lambda_l2,
'min_gain_to_split': min_gain_to_split,
'max_depth': max_depth,
'scale_pos_weight': scale_pos_weight,
'save_binary': True,
'objective': 'regression',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'mae',
'boost_from_average': False,
}
lgb_train = lgb.Dataset(train_.loc[:, column_train].values, label=train_.loc[:, 'open_channels'].values)
 lgb_valid = lgb.Dataset(val_.loc[:, column_train].values, label=val_.loc[:, 'open_channels'].values)
 clf = lgb.train(param, lgb_train, 1000, valid_sets=[lgb_valid],feval=custom_asymmetric_valid,
fobj=custom_asymmetric_train, verbose_eval=250, early_stopping_rounds = 400)
 predictions = clf.predict(val_.loc[:, column_train].values, num_iteration=clf.best_iteration)
 score = eval_gini(val_.loc[:, "open_channels"].values, predictions)
 return 1-score
bounds_lgb = {
'num_leaves':(5, 20),
'min_data_in_leaf':(5, 20),
'learning_rate':(0.01, 0.2),
'min_sum_hessian_in_leaf':(0.00001, 0.01),
'feature_fraction':(0.05, 0.5),
'lambda_l1':(0, 5.0),
'lambda_l2':(0, 5.0),
'min_gain_to_split':(0, 1.0),
'max_depth':(3,15),
'scale_pos_weight':(1, 10)
 }
lgb_bo = BayesianOptimization(lgb_bayesian, bounds_lgb, random_state=37)
 print(lgb_bo.space.keys)
 lgb_bo.maximize(init_points=5, n_iter=5, acq='ucb', xi=0.0, alpha=1e-6)
 lgb_bo.max['params']
def Create_Model() :
token_inputs = Input(( Max_length,), dtype=tf.int32, name='token_inputs')
 mask_inputs = Input(( Max_length,), dtype=tf.int32, name='mask_inputs')
 seg_inputs = Input(( Max_length,), dtype=tf.int32, name='seg_inputs')
 config = BertConfig.from_pretrained("/kaggle/input/roberta-base/config-roberta-base.json")
 model = TFRobertaModel.from_pretrained("/kaggle/input/roberta-base/pretrained-roberta-base.h5",config=config)
 pool_output,seq_out = model([token_inputs, mask_inputs, seg_inputs])
 x1 = tf.keras.layers.Dropout(0.1 )(pool_output)
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64,2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(pool_output)
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same',activation='relu' )(x2)
 x2 = tf.keras.layers.LeakyReLU(0.1 )(x2)
 x2 = tf.keras.layers.Conv1D(64,2,padding='same',activation='relu' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 bert_model1 = Model([token_inputs, mask_inputs, seg_inputs],[x1,x2])
 return bert_model1<predict_on_test>
base_param = {
'num_leaves': int(lgb_bo.max['params']['num_leaves']),
'max_bin': 63,
'min_data_in_leaf': int(lgb_bo.max['params']['min_data_in_leaf']),
'learning_rate': lgb_bo.max['params']['learning_rate'],
'min_sum_hessian_in_leaf': lgb_bo.max['params']['min_sum_hessian_in_leaf'],
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': lgb_bo.max['params']['feature_fraction'],
'lambda_l1': lgb_bo.max['params']['lambda_l1'],
'lambda_l2': lgb_bo.max['params']['lambda_l2'],
'min_gain_to_split': lgb_bo.max['params']['min_gain_to_split'],
'max_depth': int(lgb_bo.max['params']['max_depth']),
'scale_pos_weight': int(lgb_bo.max['params']['scale_pos_weight']),
'save_binary': True,
'objective': 'regression',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'mae',
'boost_from_average': False,
}
def prediction_fn(tokens_test,mask_test,seg_test,sentiment,text):
tokens_tst = []
mask_tst = []
seg_tst = []
txt = []
for i in range(0,len(tokens_test)) :
if sentiment[i] != "neutral":
tokens_tst.append(tokens_test[i])
 mask_tst.append(mask_test[i])
 seg_tst.append(seg_test[i])
 txt.append(text[i])
 tokens_tst = np.asarray(tokens_tst, dtype=np.int32)
 mask_tst = np.asarray(mask_tst, dtype=np.int32)
 seg_tst = np.asarray(seg_tst, dtype=np.int32)
 prediction = bert_model1.predict([tokens_tst,mask_tst,seg_tst],verbose=1)
 prediction1 =[]
for i in range(0,len(prediction[0])) :
a = np.argmax(prediction[0][i])
 b = np.argmax(prediction[1][i])
 if b>=a:
text1 = " "+" ".join(txt[i].split())
 enc = tokenizer.encode(text1)
 predict1 = tokenizer.decode(enc.ids[a-1:b])
 else:
predict1 = txt[i]
prediction1.append(predict1)
 return {'prediction_string':prediction1,
'prediction':prediction}<compute_test_metric>
%%time
pred_lgb, importances_lgb = model_train('lgb', base_param )
def evaluation(prediction_string,test_text,sentiment,text):
flag = 0
total_prediction_string = []
for i in range(0,len(test_text)) :
if sentiment[i] == 'neutral':
total_prediction_string.append(text[i])
 if sentiment[i] != 'neutral':
total_prediction_string.append(prediction_string[flag])
 flag+=1
jaccard_metric = []
for i in range(0,len(total_prediction_string)) :
jaccard_metric.append(jaccard(total_prediction_string[i],test_text[i]))
return sum(jaccard_metric)/len(jaccard_metric )<load_pretrained>
del train_df, test_df
gc.collect()
models_path = '/kaggle/input/roberta-model-v16/'
length = len(test[test.sentiment!='neutral'].sentiment)
 pred0 = np.zeros(( length,Max_length))
pred1 = np.zeros(( length,Max_length))
for i in range(0,5):
bert_model1 = Create_Model()
print('loading model weights')
 bert_model1.load_weights(models_path + 'v16-roberta-%i.h5'%(i))
print('making predictions')
 predictions = prediction_fn(input_tokens_test,mask_test,seg_test,test.sentiment.values,test.text.values)["prediction"]
prediction_string = prediction_fn(input_tokens_test,mask_test,seg_test,test.sentiment.values,test.text.values)["prediction_string"]
pred0 += predictions[0]/5
pred1 += predictions[1]/5<categorify>
def pred_proc(pred):
pred = np.round(np.clip(pred, 0, 10))
return pred.astype(int)
 pred = pred_proc(pred_lgb)
 set(pred )
<load_from_csv><EOS> 
submission = pd.read_csv(sub_path)
 submission['open_channels'] = pred
submission.to_csv("submission.csv", index=False, float_format='%.4f')
 submission.tail()