Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<split>
seed_random = 42
window_sizes = [10, 50]
lr_lgb = 0.05
num_leaves = 200
num_iterations = 1500
lr_wn = 0.0015
EPOCHS_wn = 15
SPLITS = 2
w_lgb = 0.5
w_wnet = 1 - w_lgb
print(w_wnet)
def train_and_evaluate_kfold(train, test, batch_size, epochs):
IMG_SIZE = 28
img_test = test.drop(['id'], axis = 1 ).values.reshape(-1, IMG_SIZE, IMG_SIZE, 1 ).astype('float32')/ 255
img_y = train['label'].values
train = train.drop(['label'], axis = 1)
 preds = np.zeros([test.shape[0], n_classes])
 for fold_n in [1, 2, 3]:
print('Training fold {}'.format(fold_n))
x_train, x_val, y_train, y_val = train_test_split(train, img_y, test_size = 0.15, random_state = 42 + fold_n)
 x_train = x_train.values.reshape(-1, IMG_SIZE, IMG_SIZE, 1 ).astype('float32')/ 255
x_val = x_val.values.reshape(-1, IMG_SIZE, IMG_SIZE, 1 ).astype('float32')/ 255
model = compile_model()
train_generator, learning_rate_reduction, checkpoint = trng_lr_ck_opt('model_fold_{}'.format(fold_n))
history = model.fit_generator(train_generator.flow(x_train, y_train, batch_size = batch_size),
steps_per_epoch = x_train.shape[0] // batch_size,
epochs = epochs,
validation_data =(x_val, y_val),
callbacks = [checkpoint, learning_rate_reduction])
 model.load_weights('model_fold_{}'.format(fold_n)+ '.hdf5')
 preds += model.predict(img_test)/ 3
preds = np.argmax(preds, axis = 1)
 return preds
preds = train_and_evaluate_kfold(train, test, BATCH_SIZE, EPOCHS )<save_to_csv>
!pip install tensorflow_addons==0.9.1
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore' )
sub['label'] = preds
sub.to_csv('cnn_kfold_baseline.csv', index = False )<import_modules>
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv' )
BertTokenizer,
AdamW,
BertModel,
BertForPreTraining,
BertConfig,
get_linear_schedule_with_warmup,
BertTokenizerFast,
RobertaModel,
RobertaTokenizerFast,
RobertaConfig 
)
 def to_list(tensor):
return tensor.detach().cpu().tolist()<categorify>
%%time
for window in window_sizes:
train["rolling_mean_" + str(window)] = train['signal'].rolling(window=window ).mean()
train["rolling_std_" + str(window)] = train['signal'].rolling(window=window ).std()
train["rolling_var_" + str(window)] = train['signal'].rolling(window=window ).var()
train["rolling_min_" + str(window)] = train['signal'].rolling(window=window ).min()
train["rolling_max_" + str(window)] = train['signal'].rolling(window=window ).max()
a =(train['signal'] - train['rolling_min_' + str(window)])/(train['rolling_max_' + str(window)] - train['rolling_min_' + str(window)])
 train["norm_" + str(window)] = a *(np.floor(train['rolling_max_' + str(window)])- np.ceil(train['rolling_min_' + str(window)]))
train = train.replace([np.inf, -np.inf], np.nan)
 train.fillna(0, inplace=True )
def process_with_offsets(args, tweet, sentiment, tokenizer):
encoded = tokenizer.encode_plus(
 sentiment,
tweet,
max_length=args.max_seq_len,
pad_to_max_length=True,
return_token_type_ids=True, 
)
 encoded["tweet"] = tweet
encoded["sentiment"] = sentiment
return encoded<feature_engineering>
%%time
for window in window_sizes:
test["rolling_mean_" + str(window)] = test['signal'].rolling(window=window ).mean()
test["rolling_std_" + str(window)] = test['signal'].rolling(window=window ).std()
test["rolling_var_" + str(window)] = test['signal'].rolling(window=window ).var()
test["rolling_min_" + str(window)] = test['signal'].rolling(window=window ).min()
test["rolling_max_" + str(window)] = test['signal'].rolling(window=window ).max()
a =(test['signal'] - test['rolling_min_' + str(window)])/(test['rolling_max_' + str(window)] - test['rolling_min_' + str(window)])
 test["norm_" + str(window)] = a *(np.floor(test['rolling_max_' + str(window)])- np.ceil(test['rolling_min_' + str(window)]))
test = test.replace([np.inf, -np.inf], np.nan)
 test.fillna(0, inplace=True )
class TweetDataset:
def __init__(self, args, tokenizer, df, mode="test"):
self.mode = mode
self.tweet = df.text.values
self.sentiment = df.sentiment.values
self.tokenizer = tokenizer
self.args = args
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
tweet = str(self.tweet[item])
 sentiment = str(self.sentiment[item])
 features = process_with_offsets(
 args=self.args,
tweet=tweet,
sentiment=sentiment,
tokenizer=self.tokenizer 
)
 return {
"input_ids":torch.tensor(features["input_ids"], dtype=torch.long),
"token_type_ids":torch.tensor(features["token_type_ids"], dtype=torch.long),
"attention_mask":torch.tensor(features["attention_mask"], dtype=torch.long),
"tweet":features["tweet"],
"sentiment":features["sentiment"]
}<normalization>
%%time
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices2'])
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels']]:
df[c+'_msignal'] = df[c] - df['signal']
df = df.replace([np.inf, -np.inf], np.nan)
 df.fillna(0, inplace=True)
 gc.collect()
return df
train = features(train)
 test = features(test )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, model_path, conf):
super(TweetModel, self ).__init__(conf)
 self.xlmroberta = transformers.XLMRobertaModel.from_pretrained(model_path, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self,input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
_,_, out = self.xlmroberta(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<categorify>
train = reduce_mem_usage(train )
def test(args, test_loader, model):
start_positions = []
end_positions = []
model.eval()
with torch.no_grad() :
t = tqdm(test_loader)
 for step, d in enumerate(t):
input_ids = d["input_ids"].to(args.device)
 attention_mask = d["attention_mask"].to(args.device)
 token_type_ids = d["token_type_ids"].to(args.device)
 logits1, logits2 = model(
 input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=None,
head_mask=None 
)
 logits1 = F.softmax(logits1, dim=1 ).cpu().data.numpy()
logits2 = F.softmax(logits2, dim=1 ).cpu().data.numpy()
start_positions.append(logits1)
 end_positions.append(logits2)
 return start_positions, end_positions<set_options>
test = reduce_mem_usage(test )
class args:
max_seq_len = 128+64
batch_size = 16
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 args.no_cuda = False if torch.cuda.is_available() else True
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()<load_from_csv>
y = train['open_channels']
col = [c for c in train.columns if c not in ['time', 'open_channels', 'group', 'medianbatch', 'abs_avgbatch', 'abs_maxbatch']]
model_path = ".. /input/xlm-roberta-base/"
config = transformers.XLMRobertaConfig.from_pretrained(model_path)
 config.output_hidden_states = True
tokenizer = transformers.XLMRobertaTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = TweetModel(model_path, config)
 model.to(args.device)
 test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv" )<create_dataframe>
test_dataset = TweetDataset(
 args=args,
df=test_df,
mode="test",
tokenizer=tokenizer 
)
 test_loader = DataLoader(
 test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
drop_last=False
)<define_variables>
model_weights_list = [
".. /input/tweet-xlm-roberta-3fold-tpu/xlm-roberta-base/xlm-roberta-base/fold_0",
".. /input/tweet-xlm-roberta-3fold-tpu/xlm-roberta-base/xlm-roberta-base/fold_1",
".. /input/tweet-xlm-roberta-3fold-tpu/xlm-roberta-base/xlm-roberta-base/fold_2"
]
start_list = []
end_list = []
for path in model_weights_list:
model.load_state_dict(torch.load(path, map_location=args.device))
start_positions, end_positions = test(args, test_loader, model)
 start_positions = np.concatenate(start_positions)
 end_positions = np.concatenate(end_positions)
 start_list.append(start_positions)
 end_list.append(end_positions )<concatenate>
start_list = np.array(start_list)
 end_list = np.array(end_list )<prepare_output>
start_positions = np.argmax(np.mean(start_list, axis=0), axis=1)
 end_positions = np.argmax(np.mean(end_list, axis=0), axis=1 )<load_from_csv>
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )
sub_df = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 for i,(text, sentiment)in enumerate(zip(test_df.text.values, test_df.sentiment.values)) :
idx_start = start_positions[i]
idx_end = end_positions[i]
encoded = tokenizer.encode_plus(
 sentiment,
text,
max_length=args.max_seq_len,
pad_to_max_length=True,
return_token_type_is=True, 
)
 input_id = encoded["input_ids"]
if idx_end < idx_start:
idx_end = idx_start
filtered_output = tokenizer.decode(input_id[idx_start:idx_end+1], skip_special_tokens=True)
 if sentiment == "neutral" or len(text.split())< 2:
filtered_output = text
filtered_output = filtered_output.strip()
sub_df.loc[i, "selected_text"] = filtered_output 
<save_to_csv>
%%time
X_train, X_valid, y_train, y_valid = train_test_split(train[col], y, test_size=0.3, random_state=seed_random)
 params = {'learning_rate': lr_lgb,
'max_depth': -1,
'num_leaves': num_leaves,
'metric': 'logloss',
'random_state': seed_random,
'n_jobs':-1,
'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(X_train, y_train), num_iterations, lgb.Dataset(X_valid, y_valid), verbose_eval=100, early_stopping_rounds=200, feval=MacroF1Metric)
 gc.collect()
sub_df.to_csv("submission.csv", index=False )<set_options>
%%time
y_lgb_pred = model.predict(test[col], num_iteration=model.best_iteration)
 y_pred_train_lgb = model.predict(train[col], num_iteration=model.best_iteration)
 gc.collect()
stoplist = stopwords.words('english')
 tqdm.pandas()
warnings.filterwarnings("ignore" )<categorify>
print('LGB score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_lgb,0,10)).astype(int), average="macro"))))
url = "http://paraphrase.org/api/en/search/"
def get_synonyms(word):
results = []
querystring = {"batchNumber":"0","needsPOSList":"true","q":word}
headers = {
'cache-control': "no-cache",
'postman-token': "2d3d31e7-b571-f4ae-d69b-8116ff64d752"
}
response = requests.request("GET", url, headers=headers, params=querystring)
 response_js = response.json()
res_count = response_js['hits']['found']
if res_count > 0:
res_count = min(3, res_count)
 hits = response_js['hits']['hit'][:res_count]
results = [ hit['target'] for hit in hits]
return results<load_from_csv>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).dropna().reset_index(drop=True)
 train = train[train.sentiment != 'neutral'].dropna().reset_index(drop=True ).head(100 )<sort_values>
vader = SentimentIntensityAnalyzer()
def sentiment_scores(word):
score = vader.polarity_scores(word)
 return score
def sort_by_len(lst):
lst.sort(key=len)
 return lst<string_transform>
EPOCHS = EPOCHS_wn
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = lr_wn
def find_synonym(text, selected_text, sentiment):
selected_texts = []
texts = []
orig_words = selected_text.split()
words = [ word_tokenize(str(word)) for word in orig_words]
words = [ word[0] for word in words if len(word)> 0]
polar_words = []
if sentiment == 'positive':
polar_words = [ word for word in words if sentiment_scores(word)['pos'] > 0]
elif sentiment == 'negative':
polar_words = [ word for word in words if sentiment_scores(word)['neg'] > 0]
if len(polar_words)== 0:
b=orig_words[0]
b=re.sub(r'\W+','',b ).lower()
for word in orig_words:
if(len(word)>len(b)) :
b=word
polar_words = [b]
polar_word = sort_by_len(polar_words)[-1]
try:
similar_words = get_synonyms(polar_word)
 for similar in similar_words:
selected_texts.append(re.sub(polar_word,similar,selected_text))
texts.append(re.sub(polar_word,similar,text))
except Exception as e:
print(e)
 if texts == [] and selected_texts == []:
return('','')
 return(texts,selected_texts )<create_dataframe>
%%time
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index'])
 return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 3:
lr = LR
elif epoch < 8:
lr = LR / 3
elif epoch < 30:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
preds_train = np.zeros(( len(train), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_train_preds = model.predict(train)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
te_train_preds = te_train_preds.reshape(-1, te_train_preds.shape[-1])
 preds_train += te_train_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 y_wnet_pred = np.argmax(preds_, axis = 1)
 y_pred_train_wnet = np.argmax(preds_train, axis = 1)
 return y_wnet_pred, y_pred_train_wnet
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 y_wnet_pred, y_pred_train_wnet = run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training Wavenet model completed...' )
<load_from_csv>
print('Wavenet score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_wnet,0,10)).astype(int), average="macro"))))
aug_df = pd.read_csv(".. /input/tse-augmented/twitter_augmented.csv")
 aug_df.head(10 )<import_modules>
def plot_cm(y_true, y_pred, title):
figsize=(14,14)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )
import os
import pandas as pd
from sklearn import model_selection
import tokenizers
import albumentations
import random
import transformers
import torch.nn as nn
import torch
from ast import literal_eval
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
from albumentations.core.transforms_interface import DualTransform, BasicTransform<set_options>
gc.collect()
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
del y_pred_train_lgb, y_pred_train_wnet
gc.collect()
seed_torch(seed=42 )<define_variables>
y_preds = w_lgb*y_lgb_pred + w_wnet*y_wnet_pred
class config:
MAX_LEN = 168
TRAIN_BATCH_SIZE = 48
VALID_BATCH_SIZE = 16
EPOCHS = 4
BASE_PATH = ".. /input/tweet-sentiment-extraction"
ROBERTA_PATH = '.. /input/roberta-base'
TRAINING_FILE = 'train_folds.csv'
AUG_FILE = '.. /input/tse-augmented/twitter_augmented.csv'
device = torch.device('cuda')
 TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True
)<save_to_csv>
del y_lgb_pred, y_wnet_pred
gc.collect()
df = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 df = df.dropna().reset_index(drop=True)
 df["kfold"] = -1
kf = model_selection.StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for fold,(trn_, val_)in enumerate(kf.split(X=df, y=df.sentiment.values)) :
print(len(trn_), len(val_))
df.loc[val_, 'kfold'] = fold
df.to_csv("train_folds.csv", index=False )<categorify>
def pred_proc(pred):
pred = np.round(np.clip(pred, 0, 10))
return pred.astype(int )
class NLPTransform(BasicTransform):

 @property
def targets(self):
return {"data": self.apply}
def update_params(self, params, **kwargs):
if hasattr(self, "interpolation"):
params["interpolation"] = self.interpolation
if hasattr(self, "fill_value"):
params["fill_value"] = self.fill_value
return params
def get_sentences(self, text, lang='en'):
return sent_tokenize(text, self.LANGS.get(lang, 'english'))
class AddSynonymTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(AddSynonymTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, selected_text = data
rand_ind = random.randint(0,2)
 text = text[rand_ind] if len(text)> rand_ind else text[0]
selected_text = selected_text[rand_ind] if len(selected_text)> rand_ind else selected_text[0]
return text, selected_text
def get_train_transforms() :
return albumentations.Compose([
AddSynonymTransform(p=0.5),
] )<string_transform>
y_preds = pred_proc(y_preds)
 y_train_preds = pred_proc(y_train_preds )
def process_data(tweet, selected_text, tweet_aug, selected_text_aug, sentiment, tokenizer, max_len, train_transforms):
if train_transforms and type(tweet_aug)!= float and len(tweet_aug)> 0:
tweet_aug, selected_text_aug = train_transforms(data=(tweet_aug, selected_text_aug)) ['data']
if type(tweet_aug)!= list:
tweet = tweet_aug
selected_text = selected_text_aug
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text, tweet_aug, selected_text_aug, train_transforms=None):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tweet_aug = tweet_aug
self.selected_text_aug = selected_text_aug
self.tokenizer = config.TOKENIZER
self.train_transforms = train_transforms
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
tweet_aug = self.tweet_aug[item] if len(self.tweet_aug)> item else []
selected_text_aug = self.selected_text_aug[item] if len(self.selected_text_aug)> item else []
data = process_data(
 self.tweet[item],
self.selected_text[item],
tweet_aug,
selected_text_aug,
self.sentiment[item],
self.tokenizer,
self.max_len,
self.train_transforms 
)
 return {
'ids': torch.tensor(data['ids'], dtype=torch.long),
'mask': torch.tensor(data['mask'], dtype=torch.long),
'token_type_ids': torch.tensor(data['token_type_ids'], dtype=torch.long),
'targets_start': torch.tensor(data['targets_start'], dtype=torch.long),
'targets_end': torch.tensor(data['targets_end'], dtype=torch.long),
'orig_tweet': data['orig_tweet'],
'orig_selected': data['orig_selected'],
'sentiment': data['sentiment'],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<train_model>
print('The main solution score {0:.4f}'.format(np.mean(f1_score(y, y_train_preds, average="macro"))))
def train_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d['ids']
token_type_ids = d['token_type_ids']
mask = d['mask']
targets_start = d['targets_start']
targets_end = d['targets_end']
sentiment = d['sentiment']
orig_selected = d['orig_selected']
orig_tweet = d['orig_tweet']
targets_start = d['targets_start']
targets_end = d['targets_end']
offsets = d['offsets']
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = utils.loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = utils.calculate_jaccard_score(
 original_tweet = tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start = np.argmax(outputs_start[px,:]),
idx_end=np.argmax(outputs_end[px,:]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 def eval_fn(data_loader, model, device):
model.eval()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d['ids']
token_type_ids = d['token_type_ids']
mask = d['mask']
targets_start = d['targets_start']
targets_end = d['targets_end']
sentiment = d['sentiment']
orig_selected = d['orig_selected']
orig_tweet = d['orig_tweet']
targets_start = d['targets_start']
targets_end = d['targets_end']
offsets = d['offsets'].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = utils.loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = utils.calculate_jaccard_score(
 original_tweet = tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start = np.argmax(outputs_start[px,:]),
idx_end=np.argmax(outputs_end[px,:]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 return jaccards.avg, losses.avg<normalization>
del y_train_preds
gc.collect()
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, model_config):
super(TweetModel, self ).__init__(model_config)
 self.roberta = transformers.RobertaModel.from_pretrained(config.ROBERTA_PATH, config=model_config)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 , 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
seq, pooled = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = self.drop_out(seq)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_train_metric>
test['open_channels'] = y_preds
test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )
class utils:
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_func = nn.CrossEntropyLoss()
start_loss = loss_func(start_logits, start_positions)
 end_loss = loss_func(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss
class AverageMeter() :

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
class EarlyStopping:
def __init__(self, patience=7, mode="max", delta=0.0003):
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score + self.delta:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
print(f'Best score till now {self.best_score}')
 def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved({} --> {} ).Saving model!'.format(self.val_score, epoch_score))
torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score<load_from_csv>
seed_random = 42
window_sizes = [10, 50]
lr_lgb = 0.05
num_leaves = 200
num_iterations = 1500
lr_wn = 0.0015
EPOCHS_wn = 15
SPLITS = 2
w_lgb = 0.5
w_wnet = 1 - w_lgb
print(w_wnet)
def run(fold):
dfx = pd.read_csv(config.TRAINING_FILE)
 aug_df = pd.read_csv(
 config.AUG_FILE,
converters={"text": literal_eval, "selected_text": literal_eval} 
)
 aug_df.columns = ['textID', 'text_aug', 'sentiment', 'selected_text_aug']
dfx = dfx.merge(aug_df, how='left')
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_train = df_train.sample(frac=1 ).reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values,
tweet_aug=df_train.text_aug.values,
selected_text_aug=df_train.selected_text_aug.values,
train_transforms= get_train_transforms() 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
shuffle=True,
num_workers=0 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values,
tweet_aug=[],
selected_text_aug=[],
train_transforms = None 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=0 
)
 device = config.device
model_config = transformers.RobertaConfig.from_pretrained(config.ROBERTA_PATH)
 model = TweetModel(model_config=model_config)
 model.to(device)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=1,
num_training_steps=num_train_steps 
)
 es = utils.EarlyStopping(patience=2, mode='max')
 print('*'* 100)
 print(f"Training is Starting for fold={fold}")
 best_jac = 0
for epoch in range(config.EPOCHS):
train_fn(train_data_loader, model, optimizer, device, scheduler)
 jaccard, loss = eval_fn(valid_data_loader, model, device)
 print(f'Jaccard Score = {jaccard}')
 print(f'Loss = {loss}')
 es(jaccard, model, model_path=f'model_{fold}.bin')
 if es.early_stop:
print('Early Stopping')
 break
if jaccard > best_jac:
best_jac = jaccard 
<define_variables>
!pip install tensorflow_addons==0.9.1
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore' )
for i in range(0, 5):
run(i )<load_from_csv>
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv' )
df_test = pd.read_csv(f"{config.BASE_PATH}/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
%%time
for window in window_sizes:
train["rolling_mean_" + str(window)] = train['signal'].rolling(window=window ).mean()
train["rolling_std_" + str(window)] = train['signal'].rolling(window=window ).std()
train["rolling_var_" + str(window)] = train['signal'].rolling(window=window ).var()
train["rolling_min_" + str(window)] = train['signal'].rolling(window=window ).min()
train["rolling_max_" + str(window)] = train['signal'].rolling(window=window ).max()
a =(train['signal'] - train['rolling_min_' + str(window)])/(train['rolling_max_' + str(window)] - train['rolling_min_' + str(window)])
 train["norm_" + str(window)] = a *(np.floor(train['rolling_max_' + str(window)])- np.ceil(train['rolling_min_' + str(window)]))
train = train.replace([np.inf, -np.inf], np.nan)
 train.fillna(0, inplace=True )
model_config = transformers.RobertaConfig.from_pretrained(config.ROBERTA_PATH)
 ENSEMBLES = [
{'model': TweetModel(model_config=model_config), 'state_dict': f"model_0.bin", 'weight': 1 },
{'model': TweetModel(model_config=model_config), 'state_dict': f"model_1.bin", 'weight': 1},
{'model': TweetModel(model_config=model_config), 'state_dict': f"model_2.bin", 'weight': 1},
{'model': TweetModel(model_config=model_config), 'state_dict': f"model_3.bin", 'weight': 1},
{'model': TweetModel(model_config=model_config), 'state_dict': f"model_4.bin", 'weight': 1},
]<load_pretrained>
%%time
for window in window_sizes:
test["rolling_mean_" + str(window)] = test['signal'].rolling(window=window ).mean()
test["rolling_std_" + str(window)] = test['signal'].rolling(window=window ).std()
test["rolling_var_" + str(window)] = test['signal'].rolling(window=window ).var()
test["rolling_min_" + str(window)] = test['signal'].rolling(window=window ).min()
test["rolling_max_" + str(window)] = test['signal'].rolling(window=window ).max()
a =(test['signal'] - test['rolling_min_' + str(window)])/(test['rolling_max_' + str(window)] - test['rolling_min_' + str(window)])
 test["norm_" + str(window)] = a *(np.floor(test['rolling_max_' + str(window)])- np.ceil(test['rolling_min_' + str(window)]))
test = test.replace([np.inf, -np.inf], np.nan)
 test.fillna(0, inplace=True )
models = []
weights = []
for val in ENSEMBLES:
model = val['model']
model.to(config.device)
 model.load_state_dict(torch.load(val['state_dict']))
model.eval()
models.append(model)
 weights.append(val['weight'] )<create_dataframe>
%%time
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices2'])
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels']]:
df[c+'_msignal'] = df[c] - df['signal']
df = df.replace([np.inf, -np.inf], np.nan)
 df.fillna(0, inplace=True)
 gc.collect()
return df
train = features(train)
 test = features(test )
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values,
tweet_aug=[],
selected_text_aug=[] 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=0 
)
 device = config.device
with torch.no_grad() :
tk0 = tqdm(test_data_loader, total=len(test_data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start = []
outputs_end = []
for index, model in enumerate(models):
output_start, output_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start.append(output_start * weights[index])
 outputs_end.append(output_end * weights[index])
 outputs_start = sum(outputs_start)/ len(outputs_start)
 outputs_end = sum(outputs_end)/ len(outputs_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = utils.calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<save_to_csv>
train = reduce_mem_usage(train )
sample = pd.read_csv(f"{config.BASE_PATH}/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.to_csv("submission.csv", index=False)
 sample.head(10 )<string_transform>
test = reduce_mem_usage(test )
stop = stopwords.words('english')
<load_from_csv>
y = train['open_channels']
col = [c for c in train.columns if c not in ['time', 'open_channels', 'group', 'medianbatch', 'abs_avgbatch', 'abs_maxbatch']]
train = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 test = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 submission = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/sample_submission.csv" )<count_missing_values>
train.isna().sum()<correct_missing_values>
train = train.dropna()<count_values>
train.sentiment.value_counts()<categorify>
def clean_text(text):

 text = str(text ).lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text<feature_engineering>
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )
train['text'] = train['text'].apply(lambda x:clean_text(x))
train['selected_text'] = train['selected_text'].apply(lambda x:clean_text(x))<feature_engineering>
%%time
X_train, X_valid, y_train, y_valid = train_test_split(train[col], y, test_size=0.3, random_state=seed_random)
 params = {'learning_rate': lr_lgb,
'max_depth': -1,
'num_leaves': num_leaves,
'metric': 'logloss',
'random_state': seed_random,
'n_jobs':-1,
'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(X_train, y_train), num_iterations, lgb.Dataset(X_valid, y_valid), verbose_eval=100, early_stopping_rounds=200, feval=MacroF1Metric)
 gc.collect()
train['temp_list'] = train['selected_text'].apply(lambda x:str(x ).split())
 top = Counter([item for sublist in train['temp_list'] for item in sublist])
 temp = pd.DataFrame(top.most_common(20))
temp.columns = ['Common_words','count']
temp.style.background_gradient(cmap='Blues' )<drop_column>
%%time
y_lgb_pred = model.predict(test[col], num_iteration=model.best_iteration)
 y_pred_train_lgb = model.predict(train[col], num_iteration=model.best_iteration)
 gc.collect()
def remove_stopword(x):
return [w for w in x if not w in stop]<feature_engineering>
print('LGB score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_lgb,0,10)).astype(int), average="macro"))))
train['temp_list'] = train['temp_list'].apply(lambda x:remove_stopword(x))<create_dataframe>
top = Counter([item for sublist in train['temp_list'] for item in sublist])
 temp = pd.DataFrame(top.most_common(20))
temp.columns = ['Common_words','count']
temp.style.background_gradient(cmap='Purples' )<statistical_test>
train_bigram =(pd.Series(nltk.ngrams(train["selected_text"], 2)).value_counts())[:20]<feature_engineering>
EPOCHS = EPOCHS_wn
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = lr_wn
train_text = train.selected_text[:100].values
generate_word_cloud(train_text )<filter>
%%time
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index'])
 return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 3:
lr = LR
elif epoch < 8:
lr = LR / 3
elif epoch < 30:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
preds_train = np.zeros(( len(train), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_train_preds = model.predict(train)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
te_train_preds = te_train_preds.reshape(-1, te_train_preds.shape[-1])
 preds_train += te_train_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 y_wnet_pred = np.argmax(preds_, axis = 1)
 y_pred_train_wnet = np.argmax(preds_train, axis = 1)
 return y_wnet_pred, y_pred_train_wnet
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 y_wnet_pred, y_pred_train_wnet = run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training Wavenet model completed...' )
positive_train = train[train["sentiment"]=="positive"]
negative_train = train[train["sentiment"]=="negative"]
neutral_train = train[train["sentiment"]=="neutral"]<string_transform>
print('Wavenet score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_wnet,0,10)).astype(int), average="macro"))))
def tokenizeandstopwords(text):
tokens = nltk.word_tokenize(text)
 token_words = [w for w in tokens if w.isalpha() ]
meaningful_words = [w for w in token_words if not w in stop]
joined_words =(" ".join(meaningful_words))
return joined_words<feature_engineering>
def plot_cm(y_true, y_pred, title):
figsize=(14,14)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )
positive_train["selected_text"] = positive_train["selected_text"].apply(clean_text)
 negative_train["selected_text"] = negative_train["selected_text"].apply(clean_text)
 neutral_train["selected_text"] = neutral_train["selected_text"].apply(clean_text )<feature_engineering>
gc.collect()
positive_train["selected_text"] = positive_train["selected_text"].apply(tokenizeandstopwords)
 negative_train["selected_text"] = negative_train["selected_text"].apply(tokenizeandstopwords)
 neutral_train["selected_text"] = neutral_train["selected_text"].apply(tokenizeandstopwords )<statistical_test>
del y_pred_train_lgb, y_pred_train_wnet
gc.collect()
positive_bigram =(pd.Series(nltk.ngrams(positive_train["selected_text"], 2)).value_counts())[:25]<feature_engineering>
y_preds = w_lgb*y_lgb_pred + w_wnet*y_wnet_pred
positive_train['temp_list'] = positive_train['selected_text'].apply(lambda x:str(x ).split())
 positive_train['temp_list'] = positive_train['temp_list'].apply(lambda x:remove_stopword(x))
positive_top = Counter([item for sublist in positive_train['temp_list'] for item in sublist])
 positive_temp = pd.DataFrame(positive_top.most_common(20))
positive_temp.columns = ['Common_words','count']
positive_temp.style.background_gradient(cmap='Blues' )<statistical_test>
del y_lgb_pred, y_wnet_pred
gc.collect()
positive_text = positive_train.selected_text[:100].values
generate_word_cloud(positive_text )<feature_engineering>
def pred_proc(pred):
pred = np.round(np.clip(pred, 0, 10))
return pred.astype(int )
negative_train['temp_list'] = negative_train['selected_text'].apply(lambda x:str(x ).split())
 negative_train['temp_list'] = negative_train['temp_list'].apply(lambda x:remove_stopword(x))
negative_top = Counter([item for sublist in negative_train['temp_list'] for item in sublist])
 negative_temp = pd.DataFrame(negative_top.most_common(20))
negative_temp.columns = ['Common_words','count']
negative_temp.style.background_gradient(cmap='Blues' )<statistical_test>
y_preds = pred_proc(y_preds)
 y_train_preds = pred_proc(y_train_preds )
negative_text = negative_train.selected_text[:100].values
generate_word_cloud(negative_text )<feature_engineering>
print('The main solution score {0:.4f}'.format(np.mean(f1_score(y, y_train_preds, average="macro"))))
neutral_train['temp_list'] = neutral_train['selected_text'].apply(lambda x:str(x ).split())
 neutral_train['temp_list'] = neutral_train['temp_list'].apply(lambda x:remove_stopword(x))
neutral_top = Counter([item for sublist in neutral_train['temp_list'] for item in sublist])
 neutral_temp = pd.DataFrame(neutral_top.most_common(20))
neutral_temp.columns = ['Common_words','count']
neutral_temp.style.background_gradient(cmap='Blues' )<statistical_test>
del y_train_preds
gc.collect()
neutral_bigram =(pd.Series(nltk.ngrams(neutral_train["selected_text"], 2)).value_counts())[:25] 
<statistical_test>
test['open_channels'] = y_preds
test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )