Kaggle competition: University of Liverpool - Ion Switching
oof_predictions["num_words_text"] = oof_predictions["text"].map(lambda x:len(x.split()))
oof_predictions["num_words_selected_text"] = oof_predictions["selected_text"].map(lambda x:len(x.split()))
oof_predictions["num_words_prediction_oof"] = oof_predictions["prediction_oof"].map(lambda x:len(x.split()))<install_modules>
import numpy as np
import pandas as pd
import gc
from sklearn.metrics import f1_score
from sklearn.model_selection import StratifiedKFold
import cudf
from cuml.ensemble import RandomForestRegressor
!pip install /kaggle/input/fairseq-and-fastbpe/sacrebleu-1.4.9-py3-none-any.whl
!pip install /kaggle/input/fairseq-and-fastbpe/fairseq-0.9.0-cp37-cp37m-linux_x86_64.whl
!pip install /kaggle/input/fairseq-and-fastbpe/fastBPE-0.1.0-cp37-cp37m-linux_x86_64.whl<set_options>
df = pd.read_csv(".. /input/remove-trends-giba-explained/train_clean_giba.csv" ).sort_values("time" ).reset_index(drop=True)
 test_df = pd.read_csv(".. /input/remove-trends-giba-explained/test_clean_giba.csv" ).sort_values("time" ).reset_index(drop=True)
 df.signal = df.signal.astype('float32')
 df.open_channels = df.open_channels.astype('float32')
 test_df.signal = test_df.signal.astype('float32' )
warnings.filterwarnings("ignore" )<define_variables>
df["group"] = np.arange(df.shape[0])//500_000
df["mg"] = np.arange(df.shape[0])//100_000
df["group"].value_counts()
LEARNING_RATE = 6e-5
MAX_LEN = 126
TRAIN_BATCH_SIZE = 35
VALID_BATCH_SIZE = 32
EPOCHS = 3
INPUT_PATH = "/kaggle/input/"
MODEL_INPUT_PATH = f"{INPUT_PATH}bertweet-model/"
key = argparse.Namespace(bpe_codes= f"{MODEL_INPUT_PATH}BERTweet_base_transformers/bpe.codes")
 bpe = fastBPE(key)
 vocab = Dictionary()
vocab.add_from_file(f"{MODEL_INPUT_PATH}BERTweet_base_transformers/dict.txt")
<string_transform>
df["category"] = 0
test_df["category"] = 0
df.loc[2_000_000:2_500_000-1, 'category'] = 1
df.loc[4_500_000:5_000_000-1, 'category'] = 1
test_df.loc[500_000:600_000-1, "category"] = 1
test_df.loc[700_000:800_000-1, "category"] = 1
df['category'] = df['category'].astype(np.float32)
 test_df['category'] = test_df['category'].astype(np.float32 )
class TweetDataset:
def __init__(self, tweets, sentiments, selected_texts):
self.tweets = [' '+' '.join(str(tweet ).split())for tweet in tweets]
self.sentiments = [' '+' '.join(str(sentiment ).split())for sentiment in sentiments]
self.selected_texts = [' '+' '.join(str(selected_text ).split())for selected_text in selected_texts]
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweets)
 def __getitem__(self, item):
e_tweet = '
 ' + bpe.encode(self.tweets[item])+ ' 
'
enc_tweet = vocab.encode_line(e_tweet, append_eos=False, add_if_not_exist=False ).long().tolist()
if self.sentiments[item] != 'neutral':
e_sentiment = '
 ' + bpe.encode(self.sentiments[item])+ " " + bpe.encode(self.sentiments[item])+ ' 
'
else:
e_sentiment = '
 ' + bpe.encode(self.sentiments[item])+ ' 
'
enc_sentiment = vocab.encode_line(e_sentiment, append_eos=False, add_if_not_exist=False ).long().tolist()
enc_tweet_sentiment = enc_tweet + enc_sentiment
padding_len = self.max_len - len(enc_tweet_sentiment)
 input_ids = enc_tweet_sentiment +([0] * padding_len)
 attention_mask =([1] * len(enc_tweet_sentiment)) +([0] * padding_len)
 start_index, end_index = 0, 0
token_type_ids = [0] * self.max_len
e_selected_text_ids = bpe.encode(self.selected_texts[item])
 enc_selected_text_ids = vocab.encode_line(e_selected_text_ids, append_eos=False, add_if_not_exist=False ).long().tolist()
for j in(i for i,e in enumerate(enc_tweet_sentiment)if e == enc_selected_text_ids[0]):
if enc_tweet_sentiment[j:j+len(enc_selected_text_ids)] == enc_selected_text_ids:
start_index = j
end_index = j+(len(enc_selected_text_ids))
return {
'ids': torch.tensor(input_ids, dtype=torch.long),
'mask': torch.tensor(attention_mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
'targets_start': torch.tensor(start_index, dtype=torch.long),
'targets_end': torch.tensor(end_index, dtype=torch.long),
'orig_tweet': self.tweets[item],
'orig_selected': self.selected_texts[item],
'sentiment': self.sentiments[item]
}<normalization>
TARGET = "open_channels"
aug_df = df[df["group"] == 5].copy()
aug_df["category"] = 1
aug_df["group"] = 10
for col in ["signal", TARGET]:
aug_df[col] += df[df["group"] == 8][col].values
aug_df['category'] = aug_df['category'].astype(np.float32)
 df = df.append(aug_df, sort=False )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(
 "/kaggle/input/bertweet-model/BERTweet_base_transformers/model.bin",
config=conf 
)
 self.drop_out = nn.Dropout(0.1)
 self.activation = nn.LeakyReLU()
self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<load_from_csv>
NUM_SHIFT = 20
features = ["signal","signal","category"]
for i in range(1, NUM_SHIFT + 1):
f_pos = "shift_{}".format(i)
 f_neg = "shift_{}".format(-i)
 features.append(f_pos)
 features.append(f_neg)
 for data in [df, test_df]:
data[f_pos] = data["signal"].shift(i ).fillna(-3 ).astype(np.float32)
 data[f_neg] = data["signal"].shift(-i ).fillna(-3 ).astype(np.float32)
 data.head()
df_test = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
%%time
NUM_FOLDS = 5
skf = StratifiedKFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 test_df = cudf.from_pandas(test_df)
 oof_preds = np.zeros(( len(df)))
 y_test = np.zeros(( len(test_df)))
 for fold,(train_ind, val_ind)in enumerate(skf.split(df, df["group"])) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
print('Fold', fold)
 train_df = cudf.from_pandas(train_df)
 val_df = cudf.from_pandas(val_df)
 model = RandomForestRegressor(
 n_estimators=35,
rows_sample = 0.35,
max_depth=18,
max_features=11,
split_algo=0,
bootstrap=False,
).fit(train_df[features], train_df[TARGET])
 pred = model.predict(val_df[features] ).to_array()
oof_preds[val_ind] = np.round(pred)
 y_test += model.predict(test_df[features] ).to_array() / NUM_FOLDS
del model; _=gc.collect()
y_test = np.round(y_test )
model_config = transformers.RobertaConfig.from_pretrained(
 f"{MODEL_INPUT_PATH}BERTweet_base_transformers/config.json" 
)
 model_config.output_hidden_states = True<create_dataframe>
f1_score(df["open_channels"], oof_preds, average="macro" )
<load_pretrained><EOS> 
test_df['time'] = [ "{0:.4f}".format(v)for v in test_df['time'].to_array() ]
test_df[TARGET] = y_test.astype(np.int32)
 test_df.to_csv('submission.csv', index=False, columns=["time", TARGET] )