Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<prepare_output>
warnings.filterwarnings("ignore" )
model.eval()
predictions = []
if check_for_images_dir() :
test_dataset = PANDADataset(
 df=test,
image_size=image_size,
n_tiles=n_tiles,
tile_mode=0 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=config.TEST_BATCH_SIZE,
shuffle=False, 
)
 for model in models:
preds = []
for idx, d in tqdm(enumerate(test_data_loader), total=len(test_data_loader)) :
inputs = d["image"]
inputs = inputs.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.to('cpu' ).numpy())
 predictions.append(np.concatenate(preds))
predictions = np.mean(predictions, axis=0)
 predictions = predictions.argmax(1)
<save_to_csv>
class config:
LEARNING_RATE = 4e-5
MAX_LEN = 192
TRAINING_FILE = ".. /input/tweet-train-folds-v2/train_8folds.csv"
ROBERTA_PATH = ".. /input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True
)
if config.DEBUG:
def quadratic_weighted_kappa(y_hat, y):
return cohen_kappa_score(y_hat, y, weights='quadratic')
 count = 0
for index, val in enumerate(test.isup_grade.values):
if predictions[index] == val:
count += 1
print(f"Accuracy Train is {(count / test.shape[0])* 100}")
 print(f'Kappa Train is {quadratic_weighted_kappa(predictions, test.isup_grade.values)}')
 else:
if len(predictions)> 0:
submission.isup_grade = predictions
submission.isup_grade = submission['isup_grade'].astype(int)
 submission.to_csv('submission.csv',index=False)
 print(submission.head() )<load_from_csv>
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]

 token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
poisson = pd.read_csv('/kaggle/input/m5-first-public-notebook-under-0-50/submission.csv')
 tweedie = pd.read_csv('/kaggle/input/m5-accuracy-tweedie-is-back/submission.csv')
 best_score = pd.read_csv('/kaggle/input/m5-accuracy-poisson-tweedie/submission(14 ).csv')
 Rv2 = pd.read_csv('/kaggle/input/m5-forecaster-v2/sub_dt_lgb.csv')
 lig = pd.read_csv('/kaggle/input/m5-forecasting-lightgbm/submission(11 ).csv' )<save_to_csv>
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }
poisson = poisson.sort_values(by = 'id' ).reset_index(drop = True)
 tweedie = tweedie.sort_values(by = 'id' ).reset_index(drop = True)
 Rv2 = Rv2.sort_values(by = 'id' ).reset_index(drop = True)
 best_score = best_score.sort_values(by = 'id' ).reset_index(drop = True)
 lig = lig.sort_values(by = 'id' ).reset_index(drop = True)
 sub = poisson.copy()
for i in sub.columns :
if i != 'id' :
sub[i] = 0.5 * best_score[i] + 0.3 * tweedie[i] + 0.2 * poisson[i]
sub.to_csv('submission.csv', index = False)
 sub.head()<load_from_csv>
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(config.ROBERTA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)

self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)

out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
poisson = pd.read_csv('/kaggle/input/m5-first-public-notebook-under-0-50/submission.csv')
 tweedie = pd.read_csv('/kaggle/input/m5-accuracy-tweedie-is-back/submission.csv')
 best_score = pd.read_csv('/kaggle/input/m5-accuracy-poisson-tweedie/submission(14 ).csv')
 Rv2 = pd.read_csv('/kaggle/input/m5-forecaster-v2/sub_dt_lgb.csv')
 lig = pd.read_csv('/kaggle/input/m5-forecasting-lightgbm/submission(11 ).csv' )<save_to_csv>
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""

 for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
poisson = poisson.sort_values(by = 'id' ).reset_index(drop = True)
 tweedie = tweedie.sort_values(by = 'id' ).reset_index(drop = True)
 Rv2 = Rv2.sort_values(by = 'id' ).reset_index(drop = True)
 best_score = best_score.sort_values(by = 'id' ).reset_index(drop = True)
 lig = lig.sort_values(by = 'id' ).reset_index(drop = True)
 sub = poisson.copy()
for i in sub.columns :
if i != 'id' :
sub[i] = 0.5 * best_score[i] + 0.3 * tweedie[i] + 0.2 * poisson[i]
sub.to_csv('submission.csv', index = False)
 sub.head()<define_variables>
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values
decay=1e-4
xtrain = []
ytrain = []
xtest = []
xval = []
yval = []
for dirname, _, filenames in os.walk('/kaggle/input/digit-recognizer/'):
for filename in filenames:
print(os.path.join(dirname, filename))
train =pd.read_csv(os.path.join(dirname,'train.csv'))
test =pd.read_csv(os.path.join(dirname,'test.csv'))
sample_submission =pd.read_csv(os.path.join(dirname,'sample_submission.csv'))
val = train[35000:]
train = train[:35000]
xtrain=train.drop('label',axis=1)
 ytrain=train.label
xtest = test
xval=val.drop('label',axis=1)
 yval=val.label
xtrain=xtrain.values
xtest=xtest.values
xval=xval.values
ytrain=ytrain.values
yval=yval.values
xtrain=xtrain.reshape(-1,28,28,1 ).astype('float32')
 xtest=xtest.reshape(-1,28,28,1 ).astype('float32')
 xval=xval.reshape(-1,28,28,1 ).astype('float32')
 xtrain=(xtrain-np.mean(xtrain)) /255
xtest=(xtest-np.mean(xtest)) /255
xval=(xval-np.mean(xval)) /255
model=Sequential()
model.add(layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)))
 model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(64,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(64,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.MaxPooling2D(2, 2))
model.add(layers.Dropout(0.2))
model.add(layers.Conv2D(128,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(128,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(128,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.MaxPooling2D(2,2))
model.add(layers.Dropout(0.2))
model.add(layers.Conv2D(256,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(256,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.MaxPooling2D(2,2))
model.add(layers.Dropout(0.2))
model.add(layers.Flatten())
 model.add(layers.Dense(256))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(256))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(10, activation='softmax'))
adam_opt = Adam(lr=0.0002, beta_1=0.9, beta_2=0.999, decay=0.0, amsgrad=False)
 optimizer = RMSprop(learning_rate=0.002,rho=0.9)
 model.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 datagen = ImageDataGenerator(
 rotation_range=12,
zoom_range=0.35,
width_shift_range=0.3,
height_shift_range=0.3, 
)
 datagen.fit(xtrain)
 learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(
 monitor='loss',
factor=0.2,
patience=2,
verbose=1,
mode="min",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001 
)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True)
 model.fit_generator(datagen.flow(xtrain, ytrain, batch_size=512),
steps_per_epoch=len(xtrain)//512,
epochs=42,
validation_data=(np.array(xval),np.array(yval)) ,
validation_steps=50,
callbacks=[learning_rate_reduction, es])
 model.summary()
score=model.evaluate(np.array(xval),np.array(yval),batch_size=512)
 results = model.predict_classes(xtest)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False)
 print("accuracy",score[1] )<set_options>
ROBERTA_PATH = ".. /input/roberta-base"
model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
device = torch.device("cuda" )
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 warnings.filterwarnings('ignore' )<load_from_csv>
%%time
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(".. /input/train-tweet-fold/model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load(".. /input/train-tweet-fold/model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load(".. /input/train-tweet-fold/model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load(".. /input/train-tweet-fold/model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load(".. /input/train-tweet-fold/model_4.bin"))
model5.eval()
df=pd.read_csv('.. /input/summeranalytics2020/train.csv')
 df.shape<set_options>
final_output = []
TEST_BATCH_SIZE = 32
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=TEST_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(outputs_start1 + outputs_start2 + outputs_start3 + outputs_start4 +
outputs_start5)/ 5
outputs_end =(outputs_end1 + outputs_end2 + outputs_end3 + outputs_end4 +
outputs_end5)/ 5
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )
<load_from_csv><EOS> 
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.to_csv("submission.csv", index=False)
 sample.head()