Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_pretrained>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 warnings.filterwarnings("ignore")
 eng_stopwords = set(stopwords.words("english"))
with open('char_pred_train_start.pkl', 'wb')as handle:
pickle.dump(probas_train_start, handle)
 with open('char_pred_train_end.pkl', 'wb')as handle:
pickle.dump(probas_train_end, handle )<feature_engineering>
train_df = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test_df = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 print("Training Shape rows = {}, columns = {}".format(train_df.shape[0],train_df.shape[1]))
print("Testing Shape rows = {}, columns = {}".format(test_df.shape[0],test_df.shape[1]))
tokenizer_char = Tokenizer(num_words=None, char_level=True, oov_token='UNK', lower=True)
 tokenizer_char.fit_on_texts(train_df['text'].values)
 len_voc = len(tokenizer_char.word_index)+ 1
X_train = tokenizer_char.texts_to_sequences(train_df['text'].values)
 X_test = tokenizer_char.texts_to_sequences(test_df['text'].values )<define_variables>
train_df.isnull().sum()
def get_start_end_string(text, selected_text):
len_selected_text = len(selected_text)
 idx_start, idx_end = 0, 0
candidates_idx = [i for i, e in enumerate(text)if e == selected_text[0]]
for idx in candidates_idx:
if text[idx : idx + len_selected_text] == selected_text:
idx_start = idx
idx_end = idx + len_selected_text-1
break
assert text[idx_start: idx_end+1] == selected_text, f'"{text[idx_start: idx_end+1]}" instead of "{selected_text}" in "{text}"'
char_targets = np.zeros(len(text))
char_targets[idx_start: idx_end+1] = 1
return idx_start, idx_end<categorify>
test_df.isnull().sum()
def load_data_second_model(df, X, char_start_probas, char_end_probas, max_len_char, train=True):
ct=len(df)
 X = pad_sequences(X, maxlen=max_len_char, padding='post', truncating='post')
 start_probas = np.zeros(( ct, max_len_char), dtype=float)
 for i, p in enumerate(char_start_probas):
len_ = min(len(p), max_len_char)
 start_probas[i, :len_] = p[:len_]
start_probas=np.expand_dims(start_probas, axis=2)
 end_probas = np.zeros(( ct, max_len_char), dtype=float)
 for i, p in enumerate(char_end_probas):
len_ = min(len(p), max_len_char)
 end_probas[i, :len_] = p[:len_]
end_probas=np.expand_dims(end_probas, axis=2)
 sentiments_list = ['positive', 'neutral', 'negative']
texts = df['text'].values
selected_texts = df['selected_text'].values if train else [''] * len(df)
 sentiments = df['sentiment'].values
sentiments_input = [[sentiments_list.index(s)]*max_len_char for s in sentiments]
if train:
start_idx = np.zeros(( ct,max_len_char),dtype='int32')
 end_idx = np.zeros(( ct,max_len_char),dtype='int32')
 for i,(text, sel_text)in enumerate(zip(df['text'].values, df['selected_text'].values)) :
start, end = get_start_end_string(text, sel_text.strip())
 start_idx[i,start]=1
end_idx[i,end]=1
else:
start_idx = [0] * len(df)
 end_idx = [0] * len(df)
 dataset= {
'ids': X,
'probas_start': start_probas,
'probas_end': end_probas,
'target_start': start_idx,
'target_end': end_idx,
'text': texts,
'selected_text': selected_texts,
'sentiment': sentiments,
'sentiment_input': np.array(sentiments_input),
}
return dataset<split>
keyword_dist = train_df.groupby("keyword")['target'].value_counts().unstack(fill_value=0)
 keyword_dist = keyword_dist.add_prefix(keyword_dist.columns.name ).rename_axis(columns=None ).reset_index()
char_dataset_train=load_data_second_model(train_df,
X_train,
probas_train_start,
probas_train_end,
max_len_char=args.max_char_length,
train=True)
 char_dataset_test=load_data_second_model(test_df,
X_test,
probas_test_start,
probas_test_end,
max_len_char=args.max_char_length,
train=False )<compute_test_metric>
keyword_dist.sort_values('target1',ascending = False ).head(10 )
LABEL_SMOOTHING=0.1
def loss_fn_2(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss<define_search_model>
keyword_dist.sort_values('target0',ascending = False ).head(10 )
class ConvBlock(L.Layer):
def __init__(self,out_dim,kernel_size,padding="same"):
super(ConvBlock, self ).__init__()
self.conv=L.Conv1D(out_dim,kernel_size, padding=padding)
 self.bn=L.BatchNormalization()
def call(self, inputs):
x=self.conv(inputs)
 x=self.bn(x)
 x=tf.keras.activations.relu(x)
 return x<choose_model_class>
train_df['word_count'] = train_df['text'].apply(lambda x : len(str(x ).split()))
test_df['word_count'] = test_df['text'].apply(lambda x : len(str(x ).split()))
train_df['unique_word_count'] = train_df['text'].apply(lambda x : len(set(str(x ).split())))
 test_df['unique_word_count'] = test_df['text'].apply(lambda x : len(set(str(x ).split())))
 train_df['count_letters'] = train_df['text'].apply(lambda x : len(str(x)))
 test_df['count_letters'] = test_df['text'].apply(lambda x : len(str(x)))
 train_df['count_punctuations'] = train_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test_df['count_punctuations'] = test_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train_df['stop_word_count'] = train_df['text'].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
test_df['stop_word_count'] = test_df['text'].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
train_df['hashtag_count'] = train_df['text'].apply(lambda x : len([c for c in str(x)if c == '
test_df['hashtag_count'] = test_df['text'].apply(lambda x : len([c for c in str(x)if c == '
train_df['mention_count'] = train_df['text'].apply(lambda x : len([c for c in str(x)if c=='@']))
test_df['mention_count'] = test_df['text'].apply(lambda x : len([c for c in str(x)if c=='@']))
class Logits(L.Layer):
def __init__(self,dim1=32,dim2=2):
super(Logits, self ).__init__()
self.dense1=L.Dense(dim1,activation='relu')
 self.dense2=L.Dense(dim2)
 def call(self, inputs):
x=self.dense1(inputs)
 x=self.dense2(x)
 return x<choose_model_class>
def generate_ngrams(text, n_gram=1):
token = [token for token in text.lower().split(' ')if token != '' if token not in eng_stopwords]
ngrams = zip(*[token[i:] for i in range(n_gram)])
 return [' '.join(ngram)for ngram in ngrams]
disaster_bigrams = defaultdict(int)
 nondisaster_bigrams = defaultdict(int)
 for tweet in train_df[train_df['target']==1]['text']:
for word in generate_ngrams(tweet, n_gram=2):
disaster_bigrams[word] += 1
for tweet in train_df[train_df['target']==0]['text']:
for word in generate_ngrams(tweet, n_gram=2):
nondisaster_bigrams[word] += 1
df_disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items() , key=lambda x: x[1])[::-1])
 df_nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items() , key=lambda x: x[1])[::-1])
 fig, axes = plt.subplots(ncols=2, figsize=(10, 10))
plt.tight_layout()
sns.barplot(y=df_disaster_bigrams[0].values[:10], x=df_disaster_bigrams[1].values[:10], ax=axes[0], color='cyan')
 sns.barplot(y=df_nondisaster_bigrams[0].values[:10], x=df_nondisaster_bigrams[1].values[:10], ax=axes[1], color='pink')
 for i in range(2):
axes[i].spines['right'].set_visible(False)
 axes[i].set_xlabel('')
 axes[i].set_ylabel('')
 axes[i].tick_params(axis='x', labelsize=10)
 axes[i].tick_params(axis='y', labelsize=10)
 axes[0].set_title('most common bigrams in Disaster Tweets', fontsize=15)
 axes[1].set_title('most common bigrams in Non-disaster Tweets', fontsize=15)
 plt.show()
class CharCNN(tf.keras.Model):
def __init__(self,len_voc,cnn_dim=32, char_embed_dim=16, sent_embed_dim=16,
proba_cnn_dim=16, kernel_size=3,max_len_char=args.max_char_length):
super(CharCNN, self ).__init__()
self.CharEmbedding = L.Embedding(input_dim=len_voc, output_dim=char_embed_dim)
 self.SentimentEmbedding = L.Embedding(input_dim=3, output_dim=sent_embed_dim)
 self.ProbasCNN = ConvBlock(proba_cnn_dim,kernel_size)
 self.CNN1=ConvBlock(cnn_dim,kernel_size)
 self.CNN2=ConvBlock(cnn_dim*2,kernel_size)
 self.CNN3=ConvBlock(cnn_dim*4,kernel_size)
 self.CNN4=ConvBlock(cnn_dim*8,kernel_size)
 self.Dropout = L.Dropout(0.5)
 self.dense_logits=Logits(32,2)
 def call(self, inputs):
char_ids,sentiment,start_probas,end_probas=inputs
probas = tf.concat([start_probas, end_probas], -1)
 probas_fts = self.ProbasCNN(probas)
 char_fts = self.CharEmbedding(char_ids)
 sentiment_fts = self.SentimentEmbedding(sentiment)
 x = tf.concat([char_fts, sentiment_fts, probas_fts], -1)
 features = self.CNN1(x)
 features = self.CNN2(features)
 features = self.CNN3(features)
 features = self.CNN4(features)
 logits=self.Dropout(features)
 logits=self.dense_logits(logits)
 start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]
start_logits= tf.keras.activations.softmax(start_logits)
 end_logits= tf.keras.activations.softmax(end_logits)
 return start_logits,end_logits<feature_engineering>
disaster_trigrams = defaultdict(int)
 nondisaster_trigrams = defaultdict(int)
 for tweet in train_df[train_df['target']==1]['text']:
for word in generate_ngrams(tweet, n_gram=3):
disaster_trigrams[word] += 1
for tweet in train_df[train_df['target']==0]['text']:
for word in generate_ngrams(tweet, n_gram=3):
nondisaster_trigrams[word] += 1
df_disaster_trigrams = pd.DataFrame(sorted(disaster_trigrams.items() , key=lambda x: x[1])[::-1])
 df_nondisaster_trigrams = pd.DataFrame(sorted(nondisaster_trigrams.items() , key=lambda x: x[1])[::-1])
 fig, axes = plt.subplots(ncols=2, figsize=(10, 10))
plt.tight_layout()
sns.barplot(y=df_disaster_trigrams[0].values[:10], x=df_disaster_trigrams[1].values[:10], ax=axes[0], color='cyan')
 sns.barplot(y=df_nondisaster_trigrams[0].values[:10], x=df_nondisaster_trigrams[1].values[:10], ax=axes[1], color='pink')
 for i in range(2):
axes[i].spines['right'].set_visible(False)
 axes[i].set_xlabel('')
 axes[i].set_ylabel('')
 axes[i].tick_params(axis='x', labelsize=10)
 axes[i].tick_params(axis='y', labelsize=10)
 axes[0].set_title('most common trigrams in Disaster Tweets', fontsize=15)
 axes[1].set_title('most common trigrams in Non-disaster Tweets', fontsize=15)
 plt.show()
inpTest=[char_dataset_test['ids'],
char_dataset_test['sentiment_input'],
char_dataset_test['probas_start'],
char_dataset_test['probas_end']]<split>
def clean(tweet):
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 tweet = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", tweet)
 punctuations = '@
for p in punctuations:
tweet = tweet.replace(p, f' {p} ')
 tweet = tweet.replace('...', '...')
 if '...' not in tweet:
tweet = tweet.replace('.. ', '...')
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 tweet = re.sub(u"\U0001F600-\U0001F64F","", tweet)
 tweet = re.sub(u"\U0001F300-\U0001F5FF","", tweet)
 tweet = re.sub(u"\U0001F680-\U0001F6FF","", tweet)
 tweet = re.sub(u"\U0001F1E0-\U0001F1FF","", tweet)
 tweet = re.sub(u"\U00002702-\U000027B0","", tweet)
 tweet = re.sub(u"\U000024C2-\U0001F251","", tweet)
 return tweet
train_df['text_cleaned'] = train_df['text'].apply(lambda s : clean(s))
test_df['text_cleaned'] = test_df['text'].apply(lambda s : clean(s))
n_splits=args.cv_splits_2
char_pred_oof_start=np.zeros(( ct_train,args.max_char_length))
char_pred_oof_end=np.zeros(( ct_train,args.max_char_length))
char_pred_test_start=np.zeros(( ct_test,args.max_char_length))
char_pred_test_end=np.zeros(( ct_test,args.max_char_length))
jac_2=[]
splits = list(StratifiedKFold(n_splits=n_splits ).split(X=train_df, y=train_df['sentiment']))
for i,(train_idx, val_idx)in enumerate(splits):
print("-----------------------------------------------------------------------")
 print(f"start train fold---{i+1}")
 inp_train=[char_dataset_train['ids'][train_idx],
char_dataset_train['sentiment_input'][train_idx],
char_dataset_train['probas_start'][train_idx],
char_dataset_train['probas_end'][train_idx]]
target_train=[char_dataset_train['target_start'][train_idx],char_dataset_train['target_end'][train_idx]]
inp_val=[char_dataset_train['ids'][val_idx],
char_dataset_train['sentiment_input'][val_idx],
char_dataset_train['probas_start'][val_idx],
char_dataset_train['probas_end'][val_idx]]
target_val=[char_dataset_train['target_start'][val_idx],char_dataset_train['target_end'][val_idx]]
model=CharCNN(len_voc)
 optimizer = tf.keras.optimizers.Adam(learning_rate=args.learning_rate_2)
 model.compile(loss=loss_fn_2,optimizer=optimizer)
 model.fit(inp_train, target_train, epochs=args.epoches_2, batch_size=128, verbose=1,
validation_data=(inp_val,target_val),
callbacks=[])
 print('preditc the oof')
 pred_val_start,pred_val_end=model.predict(inp_val,batch_size=128, verbose=1, callbacks=[])
 char_pred_oof_start[val_idx]=pred_val_start
char_pred_oof_end[val_idx]=pred_val_end
print("calcuate the jaccard for valset")
 fold_jac=[]
for k in val_idx:
a=np.argmax(char_pred_oof_start[k,])
 b=np.argmax(char_pred_oof_end[k,])
 orig_text=char_dataset_train['text'][k,]
orig_selected=char_dataset_train['selected_text'][k,]
if a>b:
pred_selected=orig_text
else:
pred_selected=orig_text[a:b+1]
jac_val=jaccard(pred_selected,orig_selected)
 fold_jac.append(jac_val)
 jac_2.append(fold_jac)
 print("fold jac is:", np.mean(fold_jac))
print('predict the testset')
 pred_test_start,pred_test_end=model.predict(inpTest,batch_size=128, verbose=1, callbacks=[])
 char_pred_test_start+=pred_test_start/n_splits
char_pred_test_end+=pred_test_end/n_splits<compute_test_metric>
def encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
print(f'The mean Jaccard value is {np.mean(jac_2)}' )<categorify>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def convert_prob_to_string(dataset, pred_start, pred_end):
ct=len(dataset['text'])
 pred=[]
for k in range(ct):
start_idx=np.argmax(pred_start[k])
 end_idx=np.argmax(pred_end[k])
 if start_idx>end_idx:
pred_selected_text=dataset['text'][k]
else:
pred_selected_text=dataset['text'][k][start_idx:end_idx+1]
pred.append(pred_selected_text.strip())
 return pred<save_to_csv>
%%time
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True )
pred=convert_prob_to_string(char_dataset_test,char_pred_test_start, char_pred_test_end)
 test_df['selected_text']=pred
test_df.to_csv('submission.csv',columns=['textID','selected_text'], index=False)
<import_modules>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
import os
import torch
import random
import statistics
import tokenizers
import numpy as np
import pandas as pd
import torch.nn as nn
from sklearn.model_selection import StratifiedKFold
from torch.utils.data import TensorDataset, DataLoader, SequentialSampler, RandomSampler
from transformers import BertPreTrainedModel, RobertaConfig, RobertaModel, AdamW
from transformers import get_cosine_schedule_with_warmup, get_linear_schedule_with_warmup<init_hyperparams>
train_input = encode(train_df.text_cleaned.values, tokenizer, max_len=160)
 test_input = encode(test_df.text_cleaned.values, tokenizer, max_len=160)
 train_labels = train_df.target.values
roberta_path = "/kaggle/input/roberta-base/"
max_len = 108
hidden_size = 768
batch_size = 32
epochs = 5
lr = 2.5e-5
dropout_rate = 0.0
hidden_dropout_prob = 0.1
attention_probs_dropout_prob = 0.2
num_classes = 2
n_splits = 5
random_seed = 0
warmup_steps = 199
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file = f"{roberta_path}vocab.json",
merges_file = f"{roberta_path}merges.txt",
lowercase=False,
add_prefix_space=False)
<set_options>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=32
)
chars = [".", "!", "?"]
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
def find_sub_list(l,sl):

 if len(sl)== 0:
return []
else:
results=[]
sll=len(sl)
 for ind in(i for i,e in enumerate(l)if e==sl[0]):
if l[ind:ind+sll]==sl:
results.append(( ind,ind+sll))
return results
def findOccurrences(s, ch):
occ = [i for i, letter in enumerate(s)if letter == ch]
return occ, len(occ)
 def preprocess_train(train_df):
train_df["preprocessed_text"] = train_df["text"]
train_df["preprocessed_selected_text"] = train_df["selected_text"]
train_df["preprocessed_text"] = train_df["preprocessed_text"].apply(lambda x: " " + x if x[0] != " " else x)
 for i,(text, selected_text)in enumerate(zip(train_df.preprocessed_text.values,
train_df.preprocessed_selected_text.values)) :
spaces = []
for char in chars:
idxs, num_idxs = findOccurrences(text, char)
 if num_idxs > 1:
for pos in range(num_idxs-2, -1, -1):
if idxs[pos] == idxs[pos+1]-1:
text = text[:idxs[pos]+1] + " " + text[idxs[pos]+1:]
spaces.append(char)
 if any(spaces):
train_df.loc[i, "preprocessed_text"] = text
for char in chars:
idxs, num_idxs = findOccurrences(selected_text, char)
 if num_idxs > 1:
for pos in range(num_idxs-2, -1, -1):
if idxs[pos] == idxs[pos+1]-1:
selected_text = selected_text[:idxs[pos]+1] + " " + selected_text[idxs[pos]+1:]
train_df.loc[i, "preprocessed_selected_text"] = selected_text
for i,(text, selected_text)in enumerate(zip(train_df.preprocessed_text.values,
train_df.preprocessed_selected_text.values)) :
text_tokenized = tokenizer.encode(text ).tokens
selected_tokenized = tokenizer.encode(selected_text ).tokens
subi_spl = find_sub_list(text_tokenized, selected_tokenized)
 selected_tokenized1 = tokenizer.encode(" " + selected_text ).tokens
subi_spl1 = find_sub_list(text_tokenized, selected_tokenized1)
 if len(subi_spl)== 0 and len(subi_spl1)> 0:
train_df.loc[i, "preprocessed_selected_text"] = " " + selected_text
for i,(text, selected_text)in enumerate(zip(train_df.preprocessed_text.values,
train_df.preprocessed_selected_text.values)) :
text_tokenized = tokenizer.encode(text ).tokens
selected_tokenized = tokenizer.encode(selected_text ).tokens
subi_spl = find_sub_list(text_tokenized, selected_tokenized)
 if len(subi_spl)== 0:
start = text.find(selected_text)
 end = start + len(selected_text)
 for s in range(start-1, -1, -1):
if text[s] != " ":
selected_text = text[s] + selected_text
else:
selected_text = " " + selected_text
break
for s in range(end, len(text)) :
if text[s] != " ":
selected_text = selected_text + text[s]
else:
break
train_df.loc[i, "preprocessed_selected_text"] = selected_text
for i,(text, selected_text)in enumerate(zip(train_df.preprocessed_text.values,
train_df.preprocessed_selected_text.values)) :
text_tokenized = tokenizer.encode(text ).tokens
selected_tokenized = tokenizer.encode(selected_text ).tokens
subi_spl = find_sub_list(text_tokenized, selected_tokenized)
 if len(subi_spl)== 0:
train_df.drop(i, inplace=True)
 train_df.reset_index(drop=True, inplace=True)
 def preprocess_test(test_df):
test_df["preprocessed_text"] = test_df["text"]
test_df["preprocessed_text"] = test_df["preprocessed_text"].apply(lambda x: " " + x if x[0] != " " else x)
 for i, text in enumerate(test_df.preprocessed_text.values):
spaces = []
for char in chars:
idxs, num_idxs = findOccurrences(text, char)
 if num_idxs > 1:
for pos in range(num_idxs-2, -1, -1):
if idxs[pos] == idxs[pos+1]-1:
text = text[:idxs[pos]+1] + " " + text[idxs[pos]+1:]
spaces.append(char)
 if any(spaces):
test_df.loc[i, "preprocessed_text"] = text
def get_labels(tokens, pr_selected_text):
start_labels = [0] * len(tokens)
 end_labels = [0] * len(tokens)
 selected_tokens = tokenizer.encode(pr_selected_text ).tokens
subi = find_sub_list(tokens, selected_tokens)
 assert len(subi)> 0, "Something is wrong!!!"
start, end = subi[0]
start_labels[start] = 1
end_labels[end-1] = 1
start_labels = [0] * 4 + start_labels + [0]
end_labels = [0] * 4 + end_labels + [0]
return start_labels, end_labels
def process_sample(text, pr_text, sentiment, max_len, tokenizer, selected_text=None, pr_selected_text=None):

 sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
info = tokenizer.encode(pr_text)
 input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + info.ids + [2]
attention_mask = [1] * len(input_ids)
 tokens = info.tokens
offsets = info.offsets
loss_mask = [0] * 4 + [1] * len(tokens)+ [0]
if selected_text is not None:
start_labels, end_labels = get_labels(tokens, pr_selected_text)
 else: start_labels, end_labels = None, None
pad_len = max_len - len(input_ids)
 if pad_len > 0:
input_ids = input_ids +([1] * pad_len)
 attention_mask = attention_mask +([0] * pad_len)
 loss_mask = loss_mask +([0] * pad_len)
 if selected_text is not None:
start_labels = start_labels +([0] * pad_len)
 end_labels = end_labels +([0] * pad_len)
 return {
"input_ids": input_ids,
"attention_mask": attention_mask,
"loss_mask": loss_mask,
"tokens": tokens,
"start_labels": start_labels,
"end_labels": end_labels,
"offsets": offsets,
"text": text,
"preprocessed_text": pr_text,
"selected_text": selected_text,
"preprocessed_selected_text": pr_selected_text,
"sentiment": sentiment
}
def tweet_loss(start_logits, start_labels, end_logits, end_labels, loss_mask):
loss_fn = nn.BCEWithLogitsLoss()
actives = loss_mask.reshape(-1)== 1
start_logits = start_logits.reshape(-1)[actives]
start_labels = start_labels.reshape(-1)[actives]
end_logits = end_logits.reshape(-1)[actives]
end_labels = end_labels.reshape(-1)[actives]
start_loss = loss_fn(start_logits, start_labels.type_as(start_logits))
end_loss = loss_fn(end_logits, end_labels.type_as(end_logits))
return(start_loss + end_loss)/ 2
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
class TweetDataset:
max_len = max_len
tokenizer = tokenizer
def __init__(self, df, sentiment_values=None):
if sentiment_values is not None:
df = df.loc[df.sentiment.isin(sentiment_values)].reset_index(drop=True)
 self.df = df
self.indexs = df.index.values
self.text = df.text.values
self.preprocessed_text = df.preprocessed_text.values
self.sentiments = df.sentiment.values
if "selected_text" in df:
self.selected_text = df.selected_text.values
self.preprocessed_selected_text = df.preprocessed_selected_text.values
else:
self.selected_text = None
self.preprocessed_selected_text = None
def __len__(self):
return len(self.indexs)
 def __getitem__(self, idx):
if self.selected_text is not None:
return process_sample(self.text[idx], self.preprocessed_text[idx], self.sentiments[idx], self.max_len,
self.tokenizer, self.selected_text[idx], self.preprocessed_selected_text[idx])
 else:
return process_sample(self.text[idx], self.preprocessed_text[idx], self.sentiments[idx], self.max_len,
self.tokenizer)
 def get_metadata(self):
return [self[i] for i in range(len(self)) ]
def get_train_dataloader(self, idxs):
input_ids_all = []
attention_mask_all = []
loss_mask_all = []
start_labels_all = []
end_labels_all = []
for i in idxs:
sample = self[i]
input_ids_all.append(sample["input_ids"])
 attention_mask_all.append(sample["attention_mask"])
 loss_mask_all.append(sample["loss_mask"])
 start_labels_all.append(sample["start_labels"])
 end_labels_all.append(sample["end_labels"])
 data = TensorDataset(torch.LongTensor(input_ids_all),
torch.FloatTensor(attention_mask_all),
torch.LongTensor(loss_mask_all),
torch.LongTensor(start_labels_all),
torch.LongTensor(end_labels_all)
)
 sampler = SequentialSampler(data)
 dataloader = DataLoader(data, sampler=sampler, batch_size=batch_size)
 return dataloader
def get_test_dataloader(self):
input_ids_all = []
attention_mask_all = []
loss_mask_all = []
for i in self.indexs:
sample = self[i]
input_ids_all.append(sample["input_ids"])
 attention_mask_all.append(sample["attention_mask"])
 loss_mask_all.append(sample["loss_mask"])
 data = TensorDataset(torch.LongTensor(input_ids_all),
torch.FloatTensor(attention_mask_all),
torch.LongTensor(loss_mask_all)
)
 sampler = SequentialSampler(data)
 dataloader = DataLoader(data, sampler=sampler, batch_size=batch_size)
 return dataloader
class TweetModel(BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = RobertaModel.from_pretrained(roberta_path, config=conf)
 self.drop_out = nn.Dropout(dropout_rate)
 self.clf = nn.Linear(hidden_size, num_classes)
 nn.init.normal_(self.clf.weight, std=0.02)
 def forward(self, input_ids, attention_mask):
sequence_outputs, _ = self.roberta(
 input_ids=input_ids,
attention_mask=attention_mask 
)
 sequence_outputs = self.drop_out(sequence_outputs)
 logits = self.clf(sequence_outputs)
 start_logits, end_logits = logits.split(1, dim=-1)
 return start_logits, end_logits
def prepare_predictions(pp):
for char in chars:
idxs, num_idxs = findOccurrences(pp, char)
 if num_idxs > 1:
for pos in range(num_idxs-2, -1, -1):
if idxs[pos] == idxs[pos+1] - 2 :
pp = pp[:idxs[pos]+1] + pp[idxs[pos+1]:]
return pp 
<load_from_csv>
model.load_weights('model.h5')
 test_pred_BERT = model.predict(test_input)
 test_pred_BERT_int = test_pred_BERT.round().astype('int' )
<train_model><EOS> 
submission['target'] = test_pred_BERT_int
submission.to_csv("submission_BERT.csv", index=False, header=True )