Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
!pip install -U tensorflow-text==2.3
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<set_options>
!pip install -U tf-models-official==2.3
if not sys.warnoptions:
warnings.simplefilter("ignore")
 tqdm.pandas()
print(transformers.__version__)
<choose_model_class>
print("TF version: ", tf.__version__ )
class PoolerStartLogits(nn.Module):
def __init__(self, config):
super().__init__()
self.dense = nn.Linear(config.hidden_size, 1)
 def forward(self, hidden_states, p_mask=None):
x = self.dense(hidden_states ).squeeze(-1)
 if p_mask is not None:
if next(self.parameters() ).dtype == torch.float16:
x = x *(1 - p_mask)- 65500 * p_mask
else:
x = x *(1 - p_mask)- 1e30 * p_mask
return x
class PoolerEndLogits(nn.Module):
def __init__(self, config):
super().__init__()
self.dense_0 = nn.Linear(config.hidden_size*2, config.hidden_size)
 self.activation = nn.Tanh()
self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps if hasattr(config,"layer_norm_eps")else 1e-5)
 self.dense_1 = nn.Linear(config.hidden_size, 1)
 def forward(self, hidden_states, start_states=None, start_positions=None, p_mask=None):
assert(
 start_states is not None or start_positions is not None 
), "One of start_states, start_positions should be not None"
if start_positions is not None:
slen, hsz = hidden_states.shape[-2:]
start_positions = start_positions[:, None, None].expand(-1, -1, hsz)
 start_states = hidden_states.gather(-2, start_positions)
 start_states = start_states.expand(-1, slen, -1)
 x = self.dense_0(torch.cat([hidden_states, start_states], dim=-1))
x = self.activation(x)
 x = self.LayerNorm(x)
 x = self.dense_1(x ).squeeze(-1)
 if p_mask is not None:
if next(self.parameters() ).dtype == torch.float16:
x = x *(1 - p_mask)- 65500 * p_mask
else:
x = x *(1 - p_mask)- 1e30 * p_mask
return x
class RobertaForSentimentExtraction(BertPreTrainedModel):
config_class = RobertaConfig
base_model_prefix = "roberta"
def __init__(self, config):
super().__init__(config)
 self.roberta = RobertaModel(config)
 self.start_logits = PoolerStartLogits(config)
 self.end_logits = PoolerEndLogits(config)
 self.init_weights()
def forward(
 self, beam_size=1, drop_head=True,
input_ids=None,attention_mask=None,token_type_ids=None,input_mask=None,position_ids=None,
head_mask=None,inputs_embeds=None,start_positions=None,end_positions=None,p_mask=None 
):
outputs = self.roberta(
 input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 if drop_head:
hidden_states = outputs[2][-3]
else:
hidden_states = outputs[0]
start_logits = self.start_logits(hidden_states, p_mask=p_mask)
 outputs = outputs[1:]
if start_positions is not None and end_positions is not None:
for x in(start_positions, end_positions):
if x is not None and x.dim() > 1:
x.squeeze_(-1)
 end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask)
 loss_fct = CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)/ 2
outputs = total_loss
else:
bsz, slen, hsz = hidden_states.size()
start_log_probs = F.softmax(start_logits, dim=-1)
 start_top_log_probs, start_top_index = torch.topk(
 start_log_probs, beam_size, dim=-1 
)
 start_top_index_exp = start_top_index.unsqueeze(-1 ).expand(-1, -1, hsz)
 start_states = torch.gather(hidden_states, -2, start_top_index_exp)
 start_states = start_states.unsqueeze(1 ).expand(-1, slen, -1, -1)
 hidden_states_expanded = hidden_states.unsqueeze(2 ).expand_as(
 start_states 
)
 p_mask = p_mask.unsqueeze(-1)if p_mask is not None else None
end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask)
 end_log_probs = F.softmax(end_logits, dim=1)
 end_top_log_probs, end_top_index = torch.topk(
 end_log_probs, beam_size, dim=1 
)
 end_top_log_probs = end_top_log_probs.view(-1, beam_size * beam_size)
 end_top_index = end_top_index.view(-1, beam_size * beam_size)
 outputs = start_top_log_probs, start_top_index, end_top_log_probs, end_top_index
return outputs
class BartForSentimentExtraction(PretrainedBartModel):
def __init__(self, config):
super().__init__(config)
 self.start_logits = PoolerStartLogits(config)
 self.end_logits = PoolerEndLogits(config)
 self.model = BartModel(config)
 self.init_weights()
def forward(
 self, beam_size=1, drop_head=True,
input_ids=None,attention_mask=None,token_type_ids=None,input_mask=None,position_ids=None,
head_mask=None,inputs_embeds=None,start_positions=None,end_positions=None,p_mask=None 
):
outputs = self.model(
 input_ids,
attention_mask=attention_mask, 
)
 if drop_head:
hidden_states = outputs[1][-3]
else:
hidden_states = outputs[0]
start_logits = self.start_logits(hidden_states, p_mask=p_mask)
 outputs = outputs[1:]
if start_positions is not None and end_positions is not None:
for x in(start_positions, end_positions):
if x is not None and x.dim() > 1:
x.squeeze_(-1)
 end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask)
 loss_fct = CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)/ 2
outputs = total_loss
else:
bsz, slen, hsz = hidden_states.size()
start_log_probs = F.softmax(start_logits, dim=-1)
 start_top_log_probs, start_top_index = torch.topk(
 start_log_probs, beam_size, dim=-1 
)
 start_top_index_exp = start_top_index.unsqueeze(-1 ).expand(-1, -1, hsz)
 start_states = torch.gather(hidden_states, -2, start_top_index_exp)
 start_states = start_states.unsqueeze(1 ).expand(-1, slen, -1, -1)
 hidden_states_expanded = hidden_states.unsqueeze(2 ).expand_as(
 start_states 
)
 p_mask = p_mask.unsqueeze(-1)if p_mask is not None else None
end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask)
 end_log_probs = F.softmax(end_logits, dim=1)
 end_top_log_probs, end_top_index = torch.topk(
 end_log_probs, beam_size, dim=1 
)
 end_top_log_probs = end_top_log_probs.view(-1, beam_size * beam_size)
 end_top_index = end_top_index.view(-1, beam_size * beam_size)
 outputs = start_top_log_probs, start_top_index, end_top_log_probs, end_top_index
return outputs<define_variables>
train_df = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 train_df.info()
train_df.head(6 )
batch_size = 64
beam_size = 3
max_sequence_length = 128<compute_train_metric>
test_df = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 test_df.info()
test_df.head(6 )
def find_best_combinations(start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, valid_start= 0, valid_end=512):
best =(valid_start, valid_end - 1)
 best_score = -9999
for i in range(len(start_top_log_probs)) :
for j in range(end_top_log_probs.shape[0]):
if valid_start <= start_top_index[i] < valid_end and valid_start <= end_top_index[j,i] < valid_end and start_top_index[i] < end_top_index[j,i]:
score = start_top_log_probs[i] * end_top_log_probs[j,i]
if score > best_score:
best =(start_top_index[i],end_top_index[j,i])
 best_score = score
return best
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 try:
return float(len(c)) /(len(a)+ len(b)- len(c))
except:
return 0
def fuzzy_match(x,y,weights=None):
l1 = len(x.split())
 matches = dict()
x_ = x.split()
if type(y)is str:
y = [y]
for curr_length in range(l1 + 1):
for i in range(l1 + 1 - curr_length):
sub_x = ' '.join(x_[i:i+curr_length])
 if sub_x not in matches:
matches[sub_x] = np.average([fuzz.ratio(sub_x,y_)for y_ in y],weights=weights)
 if len(matches)== 0:
return None, x
return matches, sorted(matches.items() , key=operator.itemgetter(1)) [-1][0]
def ensemble_v0(context, predictions,weights=None):
context = context.split()
scores = dict()
for i,j in itertools.combinations(range(len(context)+ 1),r=2):
curr_context = ' '.join(context[i:j])
 scores[curr_context] = np.average([jaccard(curr_context, p)for p in predictions],weights=weghts)
 best_score = np.max([val for val in scores.values() ])
 has_tie = np.sum([val == best_score for val in scores.values() ])> 1
if not has_tie:
for key, val in scores.items() :
if val == best_score:
return key
else:
keys = [key for key, val in scores.items() if val == best_score]
return keys[np.argmax([len(key.split())for key in keys])]
def ensemble(context, predictions,slow_mode=False,weights=None):
starts = []
ends = []
for p in predictions:
if p in context:
start = context.index(p)
 starts.append(start)
 ends.append(start+len(p))
if len(starts)== 0:
print(context)
 return ensemble_v0(context, predictions)
 scores = dict()
context = context[np.min(starts):np.max(ends)]
for i,j in itertools.combinations(range(len(context)+ 1),r=2):
if slow_mode or len(context.split())== 1 or \ 
(not(( i > 0 and context[i-1].isalnum() and context[i].isalnum())or(j < len(context)and j > 1 and context[j-1].isalnum() and context[j].isalnum()))):
curr_context = context[i:j].strip()
scores[curr_context] = np.average([jaccard(curr_context, p)for p in predictions],weights=weights)
 for pred in predictions:
if pred not in scores:
scores[pred] = np.average([jaccard(pred, p)for p in predictions],weights=weights)
 best_score = np.max([val for val in scores.values() ])
 has_tie = np.sum([val == best_score for val in scores.values() ])> 1
if not has_tie:
for key, val in scores.items() :
if val == best_score:
return key
else:
keys = [key for key, val in scores.items() if val == best_score]
return keys[np.argmax([len(key.split())for key in keys])]
basic_tokenizer = BasicTokenizer(do_lower_case=False)
 def fix_spaces(t):
for i,item in enumerate(t):
re_res = re.search('\s+$', item)
 if bool(re_res)&(i < len(t)-1):
sp = re_res.span()
t[i+1] = t[i][sp[0]:] + t[i+1]
t[i] = t[i][:sp[0]]
return t
def roberta_tokenize_v2(tokenizer, line):
tokenized_line = []
line2 = basic_tokenizer._run_split_on_punc(line)
 line2 = fix_spaces(line2)
 for item in line2:
sub_word_tokens = tokenizer.tokenize(item)
 tokenized_line += sub_word_tokens
return tokenized_line
def convert_lines(tokenizer, df, max_sequence_length = 512):
pad_token_idx = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
 outputs = np.zeros(( len(df), max_sequence_length))
type_outputs = np.zeros(( len(df), max_sequence_length))
position_outputs = np.zeros(( len(df), 2))
extracted = []
for idx, row in tqdm(df.iterrows() , total=len(df)) :
input_ids_0 = tokenizer.convert_tokens_to_ids(roberta_tokenize_v2(tokenizer, row.sentiment))
input_ids_1 = tokenizer.convert_tokens_to_ids(roberta_tokenize_v2(tokenizer, row.text))
input_ids = [tokenizer.cls_token_id, ]+ input_ids_0 + [tokenizer.sep_token_id,] +input_ids_1 + [tokenizer.sep_token_id, ]
token_type_ids = [0,]*(len(input_ids_0)+ 1)+ [1,]*(len(input_ids_1)+ 2)
 if len(input_ids)> max_sequence_length:
input_ids = input_ids[:max_sequence_length]
input_ids[-1] = tokenizer.sep_token_id
token_type_ids = token_type_ids[:max_sequence_length]
else:
input_ids = input_ids + [pad_token_idx, ]*(max_sequence_length - len(input_ids))
token_type_ids = token_type_ids + [pad_token_idx, ]*(max_sequence_length - len(token_type_ids))
assert len(input_ids)== len(token_type_ids)
 outputs[idx,:max_sequence_length] = np.array(input_ids)
 type_outputs[idx,:] = token_type_ids
return outputs, type_outputs<load_from_csv>
for df in [train_df, test_df]:
for col in ['keyword', 'location']:
df[col] = df[col].fillna(f'no_{col}' )
test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 test_df["sep_text"] = test_df.text.apply(lambda x: " ".join(x.split() ).lower())
 def get_predictions(x_test, x_type_test, model, is_xlnet=False,drop_head=True):
all_start_top_log_probs = None
all_start_top_index = None
all_end_top_log_probs = None
all_end_top_index = None
test_dataset = torch.utils.data.TensorDataset(torch.tensor(x_test,dtype=torch.long), torch.tensor(x_type_test,dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 with torch.no_grad() :
pbar = tqdm(enumerate(test_loader),total=len(test_loader),leave=False)
 for i, items in pbar:
x_batch, x_type_batch = items
attention_mask = x_batch != tokenizer.pad_token_id
p_mask = torch.zeros(x_batch.shape,dtype=torch.float32)
 p_mask[x_batch == tokenizer.pad_token_id] = 1.0
p_mask[x_batch == tokenizer.cls_token_id] = 1.0
if is_xlnet:
attention_mask = attention_mask.float()
p_mask[:,:2] = 1.0
else:
p_mask[:,:3] = 1.0
start_top_log_probs, start_top_index, end_top_log_probs, end_top_index = model(input_ids=x_batch.cuda() , attention_mask=attention_mask.cuda() , \
token_type_ids=x_type_batch.cuda() , beam_size=beam_size, p_mask=p_mask.cuda() , drop_head=drop_head)
 start_top_log_probs = start_top_log_probs.detach().cpu().numpy()
start_top_index = start_top_index.detach().cpu().numpy()
end_top_log_probs = end_top_log_probs.detach().cpu().numpy()
end_top_index = end_top_index.detach().cpu().numpy()
all_start_top_log_probs = start_top_log_probs if all_start_top_log_probs is None else np.concatenate([all_start_top_log_probs, start_top_log_probs])
 all_start_top_index = start_top_index if all_start_top_index is None else np.concatenate([all_start_top_index, start_top_index])
 all_end_top_log_probs = end_top_log_probs if all_end_top_log_probs is None else np.concatenate([all_end_top_log_probs, end_top_log_probs])
 all_end_top_index = end_top_index if all_end_top_index is None else np.concatenate([all_end_top_index, end_top_index])
 return all_start_top_log_probs,all_start_top_index,all_end_top_log_probs,all_end_top_index
def load_and_fix_state(model_path):
state_dict = torch.load(model_path)
 new_state_dict = OrderedDict()
for k, v in state_dict.items() :
name = k[7:]
new_state_dict[name] = v
return new_state_dict
def get_selected_texts(x_test, tokenizer, offset=3, is_xlnet=False):
selected_texts = []
for i_, x in tqdm(enumerate(x_test),total=len(x_test)) :
real_length = np.sum(x != tokenizer.pad_token_id)
 if is_xlnet:
real_length -= 1
best_start, best_end = find_best_combinations(all_start_top_log_probs[i_], all_start_top_index[i_], \
all_end_top_log_probs[i_].reshape(beam_size,beam_size), all_end_top_index[i_].reshape(beam_size,beam_size), \
valid_start = offset, valid_end = real_length)
 selected_text = tokenizer.decode([w for w in x[best_start:best_end] if w != tokenizer.pad_token_id],clean_up_tokenization_spaces=False)
 selected_texts.append(selected_text if selected_text in test_df.loc[i_].text
else fuzzy_match(test_df.loc[i_].text, selected_text)[-1])
 return selected_texts
def check_corrs(model_name):
corrs = np.zeros(( 5,5))
for i in range(5):
for j in range(5):
if corrs[j,i] == 0:
corrs[i,j] = np.mean([jaccard(x,y)for x,y in zip(all_preds[f"{model_name}_{i}"], all_preds[f"{model_name}_{j}"])])
 corrs[j,i] = corrs[i,j]
return corrs<load_pretrained>
mislabeledData = train_df.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 mislabeledData = mislabeledData[mislabeledData['target'] > 1]['target']
print(f"Total {mislabeledData.shape[0]} mislabled data" )
all_preds = dict()
!mkdir configs
!cp.. /input/roberta-large-quest/roberta-large-vocab.json./configs/vocab.json
!cp.. /input/bart-large/config.json./configs/config.json
!cp.. /input/roberta-large-quest/roberta-large-merges.txt./configs/merges.txt
tokenizer = BartTokenizer.from_pretrained('./configs', do_lower_case=False)
 config = BartConfig.from_pretrained('./configs',do_lower_case=False, output_hidden_states=True)
 model = BartForSentimentExtraction(config)
 model.cuda()
X_test, X_type_test = convert_lines(tokenizer, test_df, max_sequence_length= max_sequence_length)
 dirs = ["bart-drop-head","bart-drop-head-2","tweet-all"]
for data_dir in dirs:
for fn in os.listdir(f".. /input/{data_dir}"):
if "bart_" in fn:
drop_head = data_dir != "tweet-all"
print(fn,drop_head)
 model.load_state_dict(torch.load(f".. /input/{data_dir}/{fn}"), strict=True)
 model.eval()
all_start_top_log_probs,all_start_top_index,all_end_top_log_probs,all_end_top_index = get_predictions(X_test, X_type_test, model)
 selected_texts = get_selected_texts(X_test, tokenizer)
 print(selected_texts[:10])
 all_preds[f"{fn}"] = [x if type(x)is str else "" for x in selected_texts]
!rm -rf configs
!mkdir configs
!cp.. /input/roberta-base-quest/roberta-base-vocab.json./configs/vocab.json
!cp.. /input/roberta-base-quest/roberta-base-config.json./configs/config.json
!cp.. /input/roberta-base-quest/roberta-base-merges.txt./configs/merges.txt
tokenizer = RobertaTokenizer.from_pretrained(f'./configs/', do_lower_case=False)
 config = RobertaConfig.from_pretrained('./configs',do_lower_case=False, output_hidden_states=True)
 model = RobertaForSentimentExtraction(config)
 model.cuda()
X_test, X_type_test = convert_lines(tokenizer, test_df, max_sequence_length= max_sequence_length)
 dirs = ["roberta-base-drop-head","tweet-all"]
for data_dir in dirs:
for fn in os.listdir(f".. /input/{data_dir}"):
if "roberta_" in fn:
drop_head = data_dir != "tweet-all"
print(fn,drop_head)
 model.load_state_dict(torch.load(f".. /input/{data_dir}/{fn}"), strict=True)
 model.eval()
all_start_top_log_probs,all_start_top_index,all_end_top_log_probs,all_end_top_index = get_predictions(X_test, X_type_test, model)
 selected_texts = get_selected_texts(X_test, tokenizer)
 print(selected_texts[:10])
 all_preds[f"{fn}"] = [x if type(x)is str else "" for x in selected_texts]
!rm -rf configs
!mkdir configs
!cp.. /input/roberta-large-quest/roberta-large-vocab.json./configs/vocab.json
!cp.. /input/roberta-large-quest/roberta-large-config.json./configs/config.json
!cp.. /input/roberta-large-quest/roberta-large-merges.txt./configs/merges.txt
tokenizer = RobertaTokenizer.from_pretrained(f'./configs/', do_lower_case=False)
 config = RobertaConfig.from_pretrained('./configs',do_lower_case=False, output_hidden_states=True)
 model = RobertaForSentimentExtraction(config)
 model.cuda()
X_test, X_type_test = convert_lines(tokenizer, test_df, max_sequence_length= max_sequence_length)
 dirs = ["roberta-large-drop-head","roberta-large-drop-head-2","tweet-all"]
for data_dir in dirs:
for fn in os.listdir(f".. /input/{data_dir}"):
if "roberta-large_" in fn:
drop_head = data_dir != "tweet-all"
print(fn,drop_head)
 model.load_state_dict(torch.load(f".. /input/{data_dir}/{fn}"), strict=True)
 model.eval()
all_start_top_log_probs,all_start_top_index,all_end_top_log_probs,all_end_top_index = get_predictions(X_test, X_type_test, model)
 selected_texts = get_selected_texts(X_test, tokenizer)
 print(selected_texts[:10])
 all_preds[f"{fn}"] = [x if type(x)is str else "" for x in selected_texts]
!rm -rf configs 
<categorify>
train_df['target_relabeled'] = train_df['target'].copy()
train_df.loc[train_df['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0
train_df.loc[train_df['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you!
train_df.loc[train_df['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0
train_df.loc[train_df['text'] == '.POTUS
train_df.loc[train_df['text'] == 'CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1
train_df.loc[train_df['text'] == '
train_df.loc[train_df['text'] == 'In
train_df.loc[train_df['text'] == 'Who is bringing the tornadoes and floods.Who is bringing the climate change.God is after America He is plaguing her
train_df.loc[train_df['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B.Cooper.http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1
train_df.loc[train_df['text'] == "Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "wowo--=== 12000 Nigerian refugees repatriated from Cameroon", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "He came to a land which was engulfed in tribal war and turned it into a land of peace i.e.Madinah.
 train_df.loc[train_df['text'] == "Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it
train_df.loc[train_df['text'] == "The Prophet(peace be upon him)said 'Save yourself from Hellfire even if it is by giving half a date in charity.'", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "Caution: breathing may be hazardous to your health.", 'target_relabeled'] = 1
train_df.loc[train_df['text'] == "I Pledge Allegiance To The P.O.P.E.And The Burning Buildings of Epic City.??????", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "
train_df.loc[train_df['text'] == "that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time", 'target_relabeled'] = 0
train_df[train_df['text'].isin(mislabeledData.index)]
def get_features(line, tokenizer, sentiment, span=None):
MAX_LEN = 114
MAX_CHAR = 146
pad_token_id = 1
sep_token_id = 2
cls_token_id = 0
encoding = tokenizer.encode(line)
 offsets = np.array(encoding.offsets)
 token_lenghts = list(np.diff(offsets, axis =1)[:,0])
 while sum(token_lenghts)>(MAX_CHAR-4):
token_lenghts = token_lenghts[:-1]
rep_vec = [1,1,1] + token_lenghts + [1]
sentiment_id = tokenizer.encode(sentiment ).ids
input_ids = [cls_token_id] + sentiment_id + [sep_token_id] + encoding.ids[:MAX_LEN-4] + [sep_token_id]
attention_mask = [1] * len(input_ids)
 attention_padding = [0] *(MAX_LEN - len(input_ids))
padding = [pad_token_id] *(MAX_LEN - len(input_ids))
rep_vec_padding = [0] *(MAX_LEN - len(rep_vec)- 1)
 rep_vec_padding += [MAX_CHAR - sum(rep_vec_padding)- sum(rep_vec)]
input_ids.extend(padding)
 attention_mask.extend(attention_padding)
 rep_vec.extend(rep_vec_padding)
 assert len(rep_vec)== MAX_LEN
assert sum(rep_vec)== MAX_CHAR
assert len(input_ids)== MAX_LEN
assert len(attention_mask)== MAX_LEN
features = {'input_ids': np.array(input_ids,dtype=int),
'attention_mask': np.array(attention_mask,dtype=int),
'rep_vec':np.array(rep_vec,dtype=int),
}
if span is not None:
features['target'] = np.array(span,dtype=int)+(3,3)
 else:
features['target'] = np.array(( -1,-1),dtype=int)
 return features
class TEDataset(Dataset):
def __init__(self,text,tokenizer,sentiments, labels=None, verbose=False):
if labels is None:
self.features = [get_features(text[i], tokenizer,sentiments[i], span=None)for i in tqdm(range(len(text)) ,disable= 1-verbose)]
else:
self.features = [get_features(text[i], tokenizer,sentiments[i], span=labels[i])for i in tqdm(range(len(text)) ,disable= 1-verbose)]
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
return self.features[index]
def test_collate(batch):
input_dict = {}
target_dict = {}
for key in ['input_ids','attention_mask','rep_vec']:
input_dict[key] = torch.from_numpy(np.stack([b[key] for b in batch])).long()
for key in ['target']:
input_dict[key] = [None,None]
for key in ['target']:
target_dict[key] = torch.from_numpy(np.stack([b[key] for b in batch])).long()
return input_dict, target_dict
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='/kaggle/input/roberta-base-quest/roberta-base-vocab.json',
merges_file='/kaggle/input/roberta-base-quest/roberta-base-merges.txt',
lowercase=True,
add_prefix_space=False)
 test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test['text'] = test['text'].astype(str)
 test['text'] = test['text'].apply(lambda x: x.replace('ï¿½','<i>'))
test_text = test['text'].values
test_sentiments = test['sentiment'].values
te_ds = TEDataset(test_text,tokenizer,test_sentiments)
 te_dl = DataLoader(te_ds,sampler=SequentialSampler(te_ds), collate_fn=test_collate, batch_size=256)
 class TERobertaBase(nn.Module):
def __init__(self):
super(TERobertaBase, self ).__init__()
config = RobertaConfig.from_json_file('/kaggle/input/roberta-base-quest/roberta-base-config.json')
 self.transformer = RobertaModel(config)
 self.dropout = torch.nn.Dropout(0)
 hdz = self.transformer.config.hidden_size
self.rnn_start0 = nn.GRU(hdz, hdz, num_layers=4, batch_first=True, bidirectional=True)
 self.act_start0 = nn.ReLU()
self.rnn_end0 = nn.GRU(hdz, hdz, num_layers=4, batch_first=True, bidirectional=True)
 self.act_end0 = nn.ReLU()
self.head_start = nn.Sequential(nn.Linear(self.transformer.config.hidden_size,1))
self.head_end = nn.Sequential(nn.Linear(self.transformer.config.hidden_size,1))
self.beam_size = 1
def forward(self, input_dict):
beam_size = self.beam_size
input_ids= input_dict['input_ids']
attention_mask = input_dict['attention_mask']
rep_vec = input_dict['rep_vec']
outputs = self.transformer(input_ids,
attention_mask=attention_mask, 
)
 hidden_states = outputs[0]
bs = hidden_states.shape[0]
hidden_states = torch.stack([torch.repeat_interleave(hidden_states[k],rep_vec[k], axis = 0)for k in range(bs)])
 hs_start, _ = self.rnn_start0(hidden_states)
 hs_start = hs_start.reshape(hs_start.shape[0],hs_start.shape[1],hs_start.shape[2]//2,2 ).sum(-1)
 hs_start = self.act_start0(hs_start)
 hs_end, _ = self.rnn_end0(hidden_states)
 hs_end = hs_end.reshape(hs_end.shape[0],hs_end.shape[1],hs_end.shape[2]//2,2 ).sum(-1)
 hs_end = self.act_end0(hs_end)
 start_logits = self.head_start(hs_start ).squeeze(-1)
 end_logits = self.head_start(hs_end ).squeeze(-1)
 output_dict = {'start_logits':start_logits,
'end_logits':end_logits
}
return output_dict
class TERobertaLarge(nn.Module):
def __init__(self):
super(TERobertaLarge, self ).__init__()
config = RobertaConfig.from_json_file('/kaggle/input/roberta-large-quest/roberta-large-config.json')
 self.transformer = RobertaModel(config)
 self.dropout = torch.nn.Dropout(0)
 hdz = self.transformer.config.hidden_size
self.rnn_start0 = nn.GRU(hdz, hdz, num_layers=2, batch_first=True, bidirectional=True)
 self.act_start0 = nn.ReLU()
self.rnn_end0 = nn.GRU(hdz, hdz, num_layers=2, batch_first=True, bidirectional=True)
 self.act_end0 = nn.ReLU()
self.head_start = nn.Sequential(nn.Linear(self.transformer.config.hidden_size,1))
self.head_end = nn.Sequential(nn.Linear(self.transformer.config.hidden_size,1))
self.beam_size = 1
def forward(self, input_dict):
beam_size = self.beam_size
input_ids= input_dict['input_ids']
attention_mask = input_dict['attention_mask']
rep_vec = input_dict['rep_vec']
outputs = self.transformer(input_ids,
attention_mask=attention_mask, 
)
 hidden_states = outputs[0]
bs = hidden_states.shape[0]
hidden_states = torch.stack([torch.repeat_interleave(hidden_states[k],rep_vec[k], axis = 0)for k in range(bs)])
 hs_start, _ = self.rnn_start0(hidden_states)
 hs_start = hs_start.reshape(hs_start.shape[0],hs_start.shape[1],hs_start.shape[2]//2,2 ).mean(-1)
 hs_start = self.act_start0(hs_start)
 hs_end, _ = self.rnn_end0(hidden_states)
 hs_end = hs_end.reshape(hs_end.shape[0],hs_end.shape[1],hs_end.shape[2]//2,2 ).mean(-1)
 hs_end = self.act_end0(hs_end)
 start_logits = self.head_start(hs_start ).squeeze(-1)
 end_logits = self.head_start(hs_end ).squeeze(-1)
 output_dict = {'start_logits':start_logits,
'end_logits':end_logits
}
return output_dict
class TEBartLarge(nn.Module):
def __init__(self):
super(TEBartLarge, self ).__init__()
config = BartConfig.from_json_file('/kaggle/input/bart-large/config.json')
 self.transformer = BartModel(config)
 self.dropout = torch.nn.Dropout(0)
 hdz = self.transformer.config.hidden_size
self.rnn_start0 = nn.GRU(hdz, hdz, num_layers=3, batch_first=True, bidirectional=True)
 self.act_start0 = nn.ReLU()
self.rnn_end0 = nn.GRU(hdz, hdz, num_layers=3, batch_first=True, bidirectional=True)
 self.act_end0 = nn.ReLU()
self.head_start = nn.Sequential(nn.Linear(self.transformer.config.hidden_size,1))
self.head_end = nn.Sequential(nn.Linear(self.transformer.config.hidden_size,1))
self.beam_size = 1
def forward(self, input_dict):
beam_size = self.beam_size
input_ids= input_dict['input_ids']
attention_mask = input_dict['attention_mask']
rep_vec = input_dict['rep_vec']
outputs = self.transformer(input_ids,
attention_mask=attention_mask, 
)
 hidden_states = outputs[0]
bs = hidden_states.shape[0]
hidden_states = torch.stack([torch.repeat_interleave(hidden_states[k],rep_vec[k], axis = 0)for k in range(bs)])
 hs_start, _ = self.rnn_start0(hidden_states)
 hs_start = hs_start.reshape(hs_start.shape[0],hs_start.shape[1],hs_start.shape[2]//2,2 ).sum(-1)
 hs_start = self.act_start0(hs_start)
 hs_end, _ = self.rnn_end0(hidden_states)
 hs_end = hs_end.reshape(hs_end.shape[0],hs_end.shape[1],hs_end.shape[2]//2,2 ).sum(-1)
 hs_end = self.act_end0(hs_end)
 start_logits = self.head_start(hs_start ).squeeze(-1)
 end_logits = self.head_start(hs_end ).squeeze(-1)
 output_dict = {'start_logits':start_logits,
'end_logits':end_logits
}
return output_dict
def dict_unravel(batch):
input_dict, label_dict = batch
input_dict2 = {k: input_dict[k].cuda() for k in input_dict if not k == 'target'}
label_dict = {k: label_dict[k].cuda() for k in label_dict}
return input_dict2, label_dict
def get_logits(dl, model):
with torch.no_grad() :
all_start_logits = []
all_end_logits = []
for batch in tqdm(dl):
input_dict, target_dict = dict_unravel(batch)
 outs = model.forward(input_dict)
 start_logits = outs['start_logits']
end_logits = outs['end_logits']
all_start_logits += [start_logits.detach().cpu().numpy() ]
all_end_logits += [end_logits.detach().cpu().numpy() ]
all_start_logits = np.concatenate(all_start_logits)
 all_end_logits = np.concatenate(all_end_logits)
 return all_start_logits, all_end_logits
weights_path = '/kaggle/input/roberta-base-38-full-fp16-weights/'
model = TERobertaBase().cuda().eval()
test_start_logits = []
test_end_logits = []
for FOLD in [0,1,2,3,4]:
model.load_state_dict(torch.load(weights_path + f'fold{FOLD}.pt'))
test_start_logits_, test_end_logits_ = get_logits(te_dl, model)
 test_start_logits += [test_start_logits_]
test_end_logits += [test_end_logits_]
weights_path = '/kaggle/input/roberta-base-38-pseudo-full-fp16-weights/'
for FOLD in [0,1,2,3,4]:
model.load_state_dict(torch.load(weights_path + f'fold{FOLD}.pt'))
test_start_logits_, test_end_logits_ = get_logits(te_dl, model)
 test_start_logits += [test_start_logits_]
test_end_logits += [test_end_logits_]
del model
gc.collect()
weights_path = '/kaggle/input/roberta-large-3-full-fp16-weights/'
model = TERobertaLarge().cuda().eval()
for FOLD in [0,1,2,3,4]:
model.load_state_dict(torch.load(weights_path + f'fold{FOLD}.pt'))
test_start_logits_, test_end_logits_ = get_logits(te_dl, model)
 test_start_logits += [test_start_logits_]
test_end_logits += [test_end_logits_]
weights_path = '/kaggle/input/roberta-large-3-full-pseudo-weights/'
for FOLD in [0,1,2,3,4]:
model.load_state_dict(torch.load(weights_path + f'fold{FOLD}.pt'))
test_start_logits_, test_end_logits_ = get_logits(te_dl, model)
 test_start_logits += [test_start_logits_]
test_end_logits += [test_end_logits_]
del model
gc.collect()
weights_path = '/kaggle/input/bart-large-2-full-fp16-weights/'
model = TEBartLarge().cuda().eval()
for FOLD in [0,1,2,3,4]:
model.load_state_dict(torch.load(weights_path + f'fold{FOLD}.pt'))
test_start_logits_, test_end_logits_ = get_logits(te_dl, model)
 test_start_logits += [test_start_logits_]
test_end_logits += [test_end_logits_]
weights_path = '/kaggle/input/bart-large-2-full-pseudo-weights/'
for FOLD in [0,1,2,3,4]:
model.load_state_dict(torch.load(weights_path + f'fold{FOLD}.pt'))
test_start_logits_, test_end_logits_ = get_logits(te_dl, model)
 test_start_logits += [test_start_logits_]
test_end_logits += [test_end_logits_]
del model
gc.collect()
def find_best_combinations(start_logits,end_logits, valid_start= 3, valid_end=512):
best =(valid_start, valid_end-1)
 best_score = -9999
best_ = np.argmax(start_logits), np.argmax(end_logits)
 if(valid_start <= best_[0] < valid_end)and(valid_start <= best_[1] < valid_end)and(best_[0] < best_[1]):
return best_
for i in range(valid_start, valid_end):
for j in range(i + 1, valid_end):
score = start_logits[i] + end_logits[j]
if score > best_score:
best =(i,j)
 best_score = score
return best
def get_preds(all_start_logits, all_end_logits, valid_start= 3, valid_end=512):
all_start_logits = all_start_logits
start_preds = []
end_preds = []
for start_logits,end_logits in zip(all_start_logits, all_end_logits):
start, end = find_best_combinations(start_logits,end_logits, valid_start, valid_end)
 start_preds += [start]
end_preds += [end]
start_preds = np.array(start_preds)
 end_preds = np.array(end_preds)
 return start_preds, end_preds
start_logits = np.mean(test_start_logits, axis = 0)
 end_logits = np.mean(test_end_logits, axis = 0)
 test_start_preds, test_end_preds = get_preds(start_logits, end_logits, valid_start= 3, valid_end=146)
 test_pred_text = []
for i in range(len(test_text)) :
test_pred_text += [test_text[i][test_start_preds[i]-3:test_end_preds[i]-3]]<predict_on_test>
def clean_special_characters(tweet):
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 return tweet
all_vals2 = []
for i in tqdm(range(len(test_start_logits))):
test_start_preds, test_end_preds = get_preds(test_start_logits[i], test_end_logits[i], valid_start= 3, valid_end=146)
 curr_val = []
for i in range(len(test_text)) :
curr_val += [test_text[i][test_start_preds[i]-3:test_end_preds[i]-3]]
all_vals2.append(curr_val )<define_search_model>
def restore_contractions(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet
all_vals = [val for val in all_preds.values() ] + all_vals2
print(len(all_vals))
ensembled = []
sep_texts = test_df.text.values
for i in tqdm(range(len(test_df))):
if " " in sep_texts[i]:
ensembled.append(test_pred_text[i])
 else:
predictions = [val[i] for val in all_vals]
ensembled.append(ensemble(sep_texts[i], predictions, slow_mode=True)) 
<save_to_csv>
def restore_character_entity_references(tweet):
tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 return tweet
if len(all_vals)== 60:
test_df["selected_text"] = ensembled
test_df[["textID","selected_text"]].to_csv("submission.csv",index=False )<set_options>
def restore_typos_slang_and_informal_abbreviations(tweet):
tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 return tweet
warnings.filterwarnings('ignore' )<set_options>
def restore_hashtags_usernames(tweet):
tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 return tweet
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<define_variables>
def restore_acronyms(tweet):
tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 return tweet
roberta_folder = 'fine-tuning-roberta/roberta_finetuned/'<feature_engineering>
def restore_grouping_same_words_without_embeddings(tweet):
tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 return tweet
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f'.. /input/{roberta_folder}/vocab.json',
merges_file=f'.. /input/{roberta_folder}/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
def remove_urls(tweet):
tweet = re.sub("https?:\/\/t.co\/[A-Za-z0-9]*", '', tweet)
 return tweet
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 f'.. /input/{roberta_folder}/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 f'.. /input/{roberta_folder}/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.15)
 m_size = 128
self.qa_outputs1c1 = torch.nn.Conv1d(config.hidden_size, m_size, 2)
 self.qa_outputs1c2 = torch.nn.Conv1d(m_size, 64, 2)
 self.qa_outputs1 = nn.Linear(64, 1)
 self.qa_outputs2c1 = torch.nn.Conv1d(config.hidden_size, m_size, 2)
 self.qa_outputs2c2 = torch.nn.Conv1d(m_size, 64, 2)
 self.qa_outputs2 = nn.Linear(64, 1)
 self.dropout = nn.Dropout(0.15)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = torch.nn.functional.pad(x.transpose(1,2),(1, 0))
out1 = self.qa_outputs1c1(x)
 out1 = F.relu(out1)
 out1 = self.qa_outputs1c2(out1 ).transpose(1,2)
 start_logits = self.qa_outputs1(self.dropout(out1)).squeeze(-1)
 out2 = self.qa_outputs2c1(x)
 out2 = F.relu(out2)
 out2 = self.qa_outputs2c2(out2 ).transpose(1,2)
 end_logits = self.qa_outputs2(self.dropout(out2)).squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
def remove_emojis(tweet):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 tweet = emoji_pattern.sub(r'', tweet)
 return tweet
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss 
<statistical_test>
def remove_punctuations(tweet):
tweet = tweet.translate(str.maketrans('', '', string.punctuation))
return tweet
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
%%time
def clean(tweet):
tweet = clean_special_characters(tweet)
 tweet = restore_contractions(tweet)
 tweet = restore_character_entity_references(tweet)
 tweet = restore_typos_slang_and_informal_abbreviations(tweet)
 tweet = restore_hashtags_usernames(tweet)
 tweet = restore_acronyms(tweet)
 tweet = restore_grouping_same_words_without_embeddings(tweet)
 tweet = remove_urls(tweet)
 tweet = remove_emojis(tweet)
 tweet = remove_punctuations(tweet)
 tweet = re.sub("\s+", ' ', tweet)
 return tweet
train_df['text_cleaned'] = train_df['text'].apply(lambda s : clean(s))
test_df['text_cleaned'] = test_df['text'].apply(lambda s : clean(s))
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
concat_df = pd.concat([train_df, test_df], axis = 0 ).reset_index(drop = True)
 MAX_SEQ_LEN = len(max(concat_df.text_cleaned, key = len))
print('The maximum length of each sequence is:', MAX_SEQ_LEN )
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed )<import_modules>
AUTOTUNE = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 32
SEED = 42
import gc<load_from_csv>
K = 2
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
 DISASTER = train_df['target_relabeled'] == 1
print('Whole Training Set Shape = {}'.format(train_df.shape[0]))
print('Whole Training Set Unique keyword Count = {}'.format(train_df['keyword'].nunique()))
print('Whole Training Set Target Rate(Disaster/Not Disaster)= {}'.format(train_df[DISASTER]['target_relabeled'].count() /train_df[~DISASTER]['target_relabeled'].count()))
train_ds_list = []
valid_ds_list = []
for fold,(trn_idx, val_idx)in enumerate(skf.split(train_df['text_cleaned'], train_df['target_relabeled'])) :
print('
Fold {} Training Set Shape = {} - Validation Set Shape = {}'.format(fold, train_df.loc[trn_idx, 'text_cleaned'].shape[0], train_df.loc[val_idx, 'text_cleaned'].shape[0]))
print('Fold {} Training Set Unique keyword Count = {} - Validation Set Unique keyword Count = {}'.format(fold, train_df.loc[trn_idx, 'keyword'].nunique() , train_df.loc[val_idx, 'keyword'].nunique()))
print('Fold {} Training Set Target Rate(Disaster/Not Disaster)= {}'.format(fold, train_df.loc[trn_idx, 'target_relabeled'][DISASTER].count() /train_df.loc[trn_idx,'target_relabeled'][~DISASTER].count()))
trn_ds = tf.data.Dataset.from_tensor_slices(( train_df.loc[trn_idx, 'text_cleaned'].values, train_df.loc[trn_idx, 'target_relabeled'].values))
train_ds_list.append(trn_ds)
 val_ds = tf.data.Dataset.from_tensor_slices(( train_df.loc[val_idx, 'text_cleaned'].values, train_df.loc[val_idx, 'target_relabeled'].values))
valid_ds_list.append(val_ds )
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth')
 gc.collect()<load_from_csv>
test_ds_raw = tf.data.Dataset.from_tensor_slices(test_df['text_cleaned'].values)
 for text in test_ds_raw.take(5):
print(f'Review: {text}' )
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred)
<save_to_csv>
BERT_MODEL = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3'
PREPROCESS_MODEL = 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2'
print(f'BERT model selected : {BERT_MODEL}')
 print(f'Preprocess model auto-selected: {PREPROCESS_MODEL}' )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
def make_bert_preprocess_model(sentence_features, seq_length=128):

 input_segments = [
tf.keras.layers.Input(shape=() , dtype=tf.string, name=ft)
 for ft in sentence_features]
bert_preprocess = hub.load(PREPROCESS_MODEL)
 tokenizer = hub.KerasLayer(bert_preprocess.tokenize, name='tokenizer')
 segments = [tokenizer(s)for s in input_segments]
truncated_segments = segments
packer = hub.KerasLayer(bert_preprocess.bert_pack_inputs,
arguments=dict(seq_length=seq_length),
name='packer')
 model_inputs = packer(truncated_segments)
 return tf.keras.Model(input_segments, model_inputs )
print('TF version',tf.__version__)
<define_variables>
def build_classifier_model() :
inputs = dict(
 input_word_ids=tf.keras.layers.Input(shape=(None,), dtype=tf.int32, name='word_ids'),
input_mask=tf.keras.layers.Input(shape=(None,), dtype=tf.int32, name='mask'),
input_type_ids=tf.keras.layers.Input(shape=(None,), dtype=tf.int32, name='type_ids'), 
)
 encoder = hub.KerasLayer(BERT_MODEL, trainable=True, name='BERT_encoder')
 outputs = encoder(inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation=None, name='classifier' )(net)
 return tf.keras.Model(inputs, net, name='none_em_proc_model' )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
def build_classifier_model_em_proc() :
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string, name='text')
 preprocessing_layer = hub.KerasLayer(PREPROCESS_MODEL, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(BERT_MODEL, trainable=True, name='BERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation=None, name='classifier' )(net)
 return tf.keras.Model(text_input, net, name='em_proc_model' )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
train_ds = train_ds_list[0]
train_ds = train_ds.shuffle(SEED ).batch(BATCH_SIZE)
 train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
 valid_ds = valid_ds_list[0]
valid_ds = valid_ds.batch(BATCH_SIZE)
 valid_ds = valid_ds.cache().prefetch(buffer_size=AUTOTUNE )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
 metrics = tf.metrics.BinaryAccuracy()
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
EPOCHS = 10
INIT_LR = 3e-5
steps_per_epoch = tf.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * EPOCHS
num_warmup_steps = int(0.1 * num_train_steps)
 optimizer = optimization.create_optimizer(init_lr = INIT_LR,
num_train_steps = num_train_steps,
num_warmup_steps = num_warmup_steps,
optimizer_type = 'adamw' )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
model_em_proc.compile(optimizer=optimizer,
loss=loss,
metrics=metrics )
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
print('Training model with embedded preprocess model')
 history_em_proc = model_em_proc.fit(x=train_ds,
validation_data=valid_ds,
epochs = EPOCHS )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
test_ds = test_ds_raw.batch(BATCH_SIZE ).cache().prefetch(buffer_size=AUTOTUNE)
 predict_result_em_proc = tf.sigmoid(model_em_proc.predict(test_ds))
print(predict_result_em_proc )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
train_ds = train_ds_list[1]
train_ds = train_ds.shuffle(SEED ).batch(BATCH_SIZE)
 train_ds = train_ds.map(lambda x, y:(preprocess_model(x), y))
train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
 valid_ds = valid_ds_list[1]
valid_ds = valid_ds.batch(BATCH_SIZE)
 valid_ds = valid_ds.map(lambda x, y:(preprocess_model(x), y))
valid_ds = valid_ds.cache().prefetch(buffer_size=AUTOTUNE )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
steps_per_epoch = tf.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * EPOCHS
num_warmup_steps = int(0.1 * num_train_steps)
 optimizer = optimization.create_optimizer(init_lr = INIT_LR,
num_train_steps = num_train_steps,
num_warmup_steps = num_warmup_steps,
optimizer_type = 'adamw' )
print('TF version',tf.__version__ )<define_variables>
model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics )
MAX_LEN = 100
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 7
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
<load_from_csv>
print('Training model without embedded preprocess model')
 history_model = model.fit(x=train_ds,
validation_data=valid_ds,
epochs = EPOCHS )
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<load_from_csv>
test_ds = test_ds_raw.batch(BATCH_SIZE ).cache().prefetch(buffer_size=AUTOTUNE)
 test_ds = test_ds.map(lambda x: preprocess_model(x))
predict_result = tf.sigmoid(model.predict(test_ds))
print(predict_result )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 test.head()<drop_column>
result_1 = predict_result_em_proc.numpy()
result_2 = predict_result.numpy()
result = []
for i in range(len(result_1)) :
result.append(( result_1[i] + result_2[i])/2)
 result = np.round(result)
 sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 ids = sample_submission.id
final_submission = pd.DataFrame(np.c_[ids, result.astype('int')], columns = ['id', 'target'])
 final_submission.to_csv('submission.csv', index = False)
 final_submission.describe()