Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<load_from_csv>
!pip install.. /input/stochastic/stochastic-0.4.0-py2.py3-none-any.whl
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
test_df = read_test()<define_variables>
import gc
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.metrics import f1_score
from sklearn.model_selection import KFold
import lightgbm as lgb
MAX_LEN = 160
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
def jacc(y_true, y_pred, smooth=100):
intersection = K.sum(K.abs(y_true * y_pred), axis=-1)
 sum_ = K.sum(K.abs(y_true)+ K.abs(y_pred), axis=-1)
 jac =(intersection + smooth)/(sum_ - intersection + smooth)
 return(1 - jac)* smooth 
<load_pretrained>
SEED = 42
N_SPLITS = 5
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(256, 5,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(128, 5,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 5,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(5 )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(256, 5, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(128, 5, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 5, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(5 )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.15), optimizer=optimizer)
 return model 
<predict_on_test>
def reduce_memory_usage(df):
numerics = ["int16", "int32", "int64", "float64"]
for col, col_type in df.dtypes.iteritems() :
best_type = None
if col_type == "object":
df[col] = df[col].astype("category")
 best_type = "category"
elif col_type in numerics:
downcast = "integer" if "int" in str(col_type)else "float"
df[col] = pd.to_numeric(df[col], downcast=downcast)
 best_type = df[col].dtype.name
return df
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/len160/v4-'+'positive'+'-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<save_to_csv>
def macro_f1_score(preds, train_data):
labels = train_data.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average="macro")
 return "f1", score, True
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st.strip())
 test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )<import_modules>
def features(df):
df = df.sort_values(by=["time"] ).reset_index(drop=True)
 df.index = df.time * 10000 - 1
df["batch"] = df.index // 25000
df["batch_index"] = df.index - df.batch * 25000
df["batch_slices"] = df["batch_index"] // 2500
df["batch_slices_2"] = df["batch"].astype(str)+ "_" + df["batch_slices"].astype(str)
 for c in ["batch", "batch_slices_2"]:
d = {}
d["mean_" + c] = df.groupby([c])["signal"].mean()
d["std_" + c] = df.groupby([c])["signal"].std()
d["median_" + c] = df.groupby([c])["signal"].median()
d["min_" + c] = df.groupby([c])["signal"].min()
d["max_" + c] = df.groupby([c])["signal"].max()
d["mean_abs_change_" + c] = df.groupby([c])["signal"].apply(lambda x: np.mean(np.abs(np.diff(x))))
d["abs_max_" + c] = df.groupby([c])["signal"].apply(lambda x: np.max(np.abs(x)))
 d["abs_min_" + c] = df.groupby([c])["signal"].apply(lambda x: np.min(np.abs(x)))
 d["range_" + c] = d["max_" + c] - d["min_" + c]
d["abs_range_" + c] = np.abs(d["max_" + c] - d["min_" + c])
 d["max_to_min_" + c] = d["max_" + c] / d["min_" + c]
d["abs_avg_" + c] =(d["abs_min_" + c] + d["abs_max_" + c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df["signal_shift_+1"] = [0] + list(df["signal"].values[:-1])
 df["signal_shift_-1"] = list(df["signal"].values[1:])+ [0]
for i in df[df["batch_index"] == 0].index:
df["signal_shift_+1"][i] = np.nan
for i in df[df["batch_index"] == 49999].index:
df["signal_shift_-1"][i] = np.nan
for c in [x for x in df.columns if x not in ["time", "signal", "open_channels", "batch", "batch_index", "batch_slices", "batch_slices_2"]]:
df[c + "_minus_signal"] = df[c] - df["signal"]
return df
print('TF version',tf.__version__ )<define_variables>
train = pd.read_csv("/kaggle/input/liverpool-ion-switching/train.csv")
 test = pd.read_csv("/kaggle/input/liverpool-ion-switching/test.csv" )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<count_values>
train["signal"] = np.exp(train["signal"])
 test["signal"] = np.exp(test["signal"] )
train["sentiment"].value_counts()<feature_engineering>
%%time
train = features(train)
 test = features(test )
train_dict = dict()
train_dict_index = dict()
for sentiment in all_sentiments:
train_dict[sentiment] = train[train["sentiment"]==sentiment]
train_dict_index[sentiment] = list(train_dict[sentiment].index)
 train_dict[sentiment] = train_dict[sentiment].reset_index(drop=True )<define_variables>
%%time
train = reduce_memory_usage(train)
 test = reduce_memory_usage(test )
def get_training_data(train):
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
return input_ids, attention_mask, token_type_ids, start_tokens, end_tokens<prepare_x_and_y>
features = [x for x in train.columns if x not in ["time", "open_channels", "batch", "batch_index", "batch_slices", "batch_slices_2"]]
target = "open_channels"
train_data_dict = dict()
for sentiment in all_sentiments:
train_data_dict[sentiment] = get_training_data(train_dict[sentiment] )<load_from_csv>
X, y = train[features], train[target]
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('' )<feature_engineering>
del train
gc.collect()
test_dict = dict()
test_dict_index = dict()
for sentiment in all_sentiments:
test_dict[sentiment] = test[test["sentiment"]==sentiment]
test_dict_index[sentiment] = list(test_dict[sentiment].index)
 test_dict[sentiment] = test_dict[sentiment].reset_index(drop=True )<categorify>
params = {
"learning_rate": 0.1,
"max_depth": -1,
"num_leaves": 2**7 + 1,
"metric": "l2",
"random_state": SEED,
"n_jobs": -1,
"sample_fraction": 0.33
}
def get_test_data(test):
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
return input_ids_t, attention_mask_t, token_type_ids_t<feature_engineering>
folds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 preds = np.zeros(len(test))
test_data_dict = dict()
for sentiment in all_sentiments:
test_data_dict[sentiment] = get_test_data(test_dict[sentiment] )<load_pretrained>
feature_importance = pd.DataFrame()
feature_importance["Feature"] = X.columns
feature_importance["Value"] = 0
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<feature_engineering>
for fold,(train_idx, test_idx)in enumerate(folds.split(X)) :
print("Fold", fold)
 X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]
X_test, y_test = X.iloc[test_idx], y.iloc[test_idx]
model = lgb.train(params, train_set=lgb.Dataset(X_train, y_train), num_boost_round=2000, valid_sets=lgb.Dataset(X_test, y_test), feval=macro_f1_score, early_stopping_rounds=100, verbose_eval=100)
 preds += model.predict(test[features], num_iteration=model.best_iteration)
 current_importance = pd.DataFrame(zip(X.columns, model.feature_importance()), columns=["Feature", "Value"])
 feature_importance = pd.concat(( feature_importance, current_importance)).groupby("Feature", as_index=False ).sum()
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<compute_test_metric>
preds = preds / N_SPLITS
def find_a_b(start_array, end_array):
a = np.argmax(start_array)
 b = np.argmax(end_array)
 if a<b:
return a,b
else:
if a<len(start_array)-1:
end_array_copy = np.array(end_array)
 end_array_copy[b]=0.0
return find_a_b(start_array, end_array_copy)
 else:
if b<len(start_array)-1:
start_array_copy = np.array(start_array)
 start_array_copy[a]=0.0
return find_a_b(start_array_copy, end_array)
 else:
return 0, len(end_array)-1<define_variables>
test["open_channels"] = np.round(np.clip(preds, 0, 10)).astype(int)
 test[["time", "open_channels"]].to_csv("submission.csv", index=False, float_format="%.4f" )
def train_model_for_sentiment(train_data_dict, test_data_dict, sentiment, train_dict):
input_ids, attention_mask, token_type_ids, start_tokens, end_tokens = train_data_dict[sentiment]
input_ids_t, attention_mask_t, token_type_ids_t = test_data_dict[sentiment]
train = train_dict[sentiment]
jac = []; VER='v1'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 print('
print('
print('
for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i-%s.h5'%(VER,fold,sentiment), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i-%s.h5'%(VER,fold,sentiment))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a,b = find_a_b(oof_start[k,], oof_end[k,])
 text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
return preds_start, preds_end<predict_on_test>
feature_importance.sort_values("Value", ascending=False ).to_csv("feature_importance.csv", index=False )