Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn.model_selection as ms
import seaborn as sns
import sklearn.linear_model as lm
import sklearn.metrics as m
import sklearn.dummy as d
import sklearn.preprocessing as p
import sklearn.ensemble as e
import sklearn.pipeline as pipeline
import sklearn.svm as svm
import sklearn.impute as impute
from sklearn.compose import make_column_transformer
import scipy.stats as stats
import lightgbm as lgb
import xgboost as xgb
train_fastext_oov,train_fastext_coverage,train_fastext_text = check_embedding_coverage(train_df["text_cleaned"],fasttext_embeddings)
 test_fastext_oov,test_fastext_coverage,test_fastext_text = check_embedding_coverage(test_df["text_cleaned"],fasttext_embeddings)
 print("FastText embedding cover {} of vocabulary and {} of text in the training dataset".format(train_fastext_coverage,train_fastext_text))
print("FastText embedding cover {} of vocabularly and {} of text in the test dataset".format(test_fastext_coverage,test_fastext_text))<drop_column>
plt.figure(figsize=(3,3))
del(train_fastext_oov)
 del(test_fastext_oov)
 del(train_glove_oov)
 del(test_glove_oov)
 del(glove_embeddings)
 del(fasttext_embeddings )<sort_values>
sns.set(rc={'figure.figsize':(20,15)} )
missalabeled_text = train_df.groupby("text" ).nunique().sort_values(by="target",ascending =False)
 df =missalabeled_text[missalabeled_text["target"] > 1]
df.index.tolist()<feature_engineering>
hp = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv' )
train_df["rebuild_target"] = train_df["target"].copy()
train_df.loc[train_df["text"]=="like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit","rebuild_target"]=0
train_df.loc[train_df["text"]=="Hellfire! We don\x89Ûªt even want to think about it or mention it so let\x89Ûªs not do anything that leads to it
train_df.loc[train_df["text"]=="The Prophet(peace be upon him)said 'Save yourself from Hellfire even if it is by giving half a date in charity.'","rebuild_target"]=0
train_df.loc[train_df["text"]=="In
train_df.loc[train_df["text"]=="To fight bioterrorism sir.","rebuild_target"]=0
train_df.loc[train_df["text"]=="Who is bringing the tornadoes and floods.Who is bringing the climate change.God is after America He is plaguing her
train_df.loc[train_df["text"]=="
train_df.loc[train_df["text"]=="
train_df.loc[train_df["text"]=="He came to a land which was engulfed in tribal war and turned it into a land of peace i.e.Madinah.
 train_df.loc[train_df["text"]=="RT NotExplained: The only known image of infamous hijacker D.B.Cooper.http://t.co/JlzK2HdeTG","rebuild_target"]=0
train_df.loc[train_df["text"]=="Hellfire is surrounded by desires so be careful and don\x89Ûªt let your desires control you!
train_df.loc[train_df["text"]=="CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring","rebuild_target"]=0
train_df.loc[train_df["text"]=="Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...","rebuild_target"]=0
train_df.loc[train_df["text"]=="wowo--=== 12000 Nigerian refugees repatriated from Cameroon","rebuild_target"]=0
train_df.loc[train_df["text"]==".POTUS
train_df.loc[train_df["text"]=="Caution: breathing may be hazardous to your health.","rebuild_target"]=0
train_df.loc[train_df["text"]=="I Pledge Allegiance To The P.O.P.E.And The Burning Buildings of Epic City.??????","rebuild_target"]=0
train_df.loc[train_df["text"]=="that horrible sinking feeling when you\x89Ûªve been at home on your phone for a while and you realise its been on 3G this whole time","rebuild_target"]=0<count_unique_values>
def print_types_in_df(df):
types = ['object', 'category', 'int64', 'float64']
types_to_counts = {x: len(df.select_dtypes(include=[x] ).columns.values)for x in types}
print(types_to_counts)
 print('sum: ' +str(sum(types_to_counts.values())) )
k=2
SEED= 1337
sk = StratifiedKFold(n_splits=k,random_state=SEED,shuffle=True)
 Disaster = train_df["target"] == 1
print("Whole Training Set Shape = {}".format(train_df.shape))
print("Whole Training Set Unique keyword Count = {}".format(train_df["keyword"].nunique()))
print("Whole training Set Target Rate(Disaster){}/{}".format(train_df[Disaster]["rebuild_target"].count() ,\
train_df[~Disaster]["rebuild_target"].count()))
for fold,(tr_idx,val_idx)in enumerate(sk.split(train_df["text_cleaned"],train_df["rebuild_target"]),\
1):
print("
Fold {} Training Set Shape={} - Validation Set Shape={}".format(fold,train_df.loc[tr_idx,"text_cleaned"].shape,\
train_df.loc[val_idx,"text_cleaned"].shape))
print("Fold {} Training Set unique keywords count{} - Validation Set unique keywords count{}".format(fold,\
train_df.loc[tr_idx,"keyword"].nunique() ,train_df.loc[val_idx,"keyword"]\ 
.nunique())) 
<train_model>
categorical_oh_features = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle',
'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'Heating', 'Electrical', 'Functional', 'GarageType', 'MiscFeature', 'SaleType', 'SaleCondition']
categorical_l_features = ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'CentralAir', 'KitchenQual', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']
numerical_features = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']
date_features = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MoSold', 'YrSold']
unused_features = ['Id']
target_feature = ['SalePrice']
class ClassificationReport(Callback):
def __init__(self,train_data=() ,val_data=()):
super(Callback,self ).__init__()
self.X_train,self.y_train = train_data
self.train_precision_scores = []
self.train_recall_scores = []
self.train_f1_scores = []
self.X_val,self.y_val = val_data
self.val_precision_scores = []
self.val_recall_scores = []
self.val_f1_scores = []
def on_epoch_end(self,epoch,logs={}):
train_prediction = np.round(self.model.predict(self.X_train,verbose=0))
train_precision = precision_score(self.y_train,train_prediction,average="macro")
 train_recall = recall_score(self.y_train,train_prediction,average="macro")
 train_f1 = f1_score(self.y_train,train_prediction,average="macro")
 self.train_precision_scores.append(train_precision)
 self.train_recall_scores.append(train_recall)
 self.train_f1_scores.append(train_f1)
 val_prediction = np.round(self.model.predict(self.X_val,verbose=0))
val_precision = precision_score(self.y_val,val_prediction,average="macro")
 val_recall = recall_score(self.y_val,val_prediction,average="macro")
 val_f1 = f1_score(self.y_val,val_prediction,average="macro")
 self.val_precision_scores.append(val_precision)
 self.val_recall_scores.append(val_recall)
 self.val_f1_scores.append(val_f1)
 print("
Epoch: {} - Training Precision: {} - Training Recall: {} - Training F1: {}".format(\
epoch+1,train_precision,train_recall,train_f1))
print("
Epoch: {} - Validation Precision: {} - Validation Recall: {} - Validation F1: {}".format(\
epoch+1,val_precision,val_recall,val_f1)) 
<choose_model_class>
len(categorical_oh_features)+ len(categorical_l_features)+ len(numerical_features)+ len(date_features)+ len(unused_features)+ len(target_feature )
class BertDisasterDetecter:
def __init__(self,max_seq_length=128,epoch=100,batch_size=128,lr=1e-3):
self.bert=hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1",
trainable=True)
 self.epoch = epoch
self.lr = lr
self.max_seq_length = max_seq_length
self.batch_size = batch_size
vocab_file = self.bert.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = self.bert.resolved_object.do_lower_case.numpy()
self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 self.models = []
self.scores = {}
def encode(self,texts):
all_tokens = []
all_mask = []
all_segment = []
for text in texts:
tx= self.tokenizer.tokenize(text)
 tx = ['[CLS]']+tx[:self.max_seq_length - 2] + ['[SEP]']
pad_len = self.max_seq_length - len(tx)
 tokens = self.tokenizer.convert_tokens_to_ids(tx)
 tokens += [0] * pad_len
mask_ids = [1]*len(tx)+ [0]*pad_len
segments_ids = [0] * self.max_seq_length
all_tokens.append(tokens)
 all_mask.append(mask_ids)
 all_segment.append(segments_ids)
 return np.array(all_tokens),np.array(all_mask),np.array(all_segment)
 def build_model(self):
input_word_ids = tf.keras.layers.Input(shape=(self.max_seq_length,), dtype=tf.int32,
name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(self.max_seq_length,), dtype=tf.int32,
name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(self.max_seq_length,), dtype=tf.int32,
name="segment_ids")
 pooled_output, sequence_output = self.bert([input_word_ids, input_mask, segment_ids])
 x = sequence_output[:,0,:]
output = Dense(1,activation="sigmoid" )(x)
 model =Model(inputs=[input_word_ids,input_mask,segment_ids],outputs=output)
 model.compile(optimizer =SGD(self.lr,momentum=0.8),loss="binary_crossentropy",metrics=['accuracy'])
 return model
def train(self):
for fold,(train_ind,test_ind)in enumerate(sk.split(train_df["text_cleaned"],train_df["keyword"])) :
print("
Fold {}
".format(fold))
Xtr = train_df.loc[train_ind,"text_cleaned"].str.lower()
Ytr = train_df.loc[train_ind,"rebuild_target"]
Xval = train_df.loc[test_ind,"text_cleaned"].str.lower()
Yval = train_df.loc[test_ind,"rebuild_target"]
Xtr_encoded = self.encode(Xtr)
 Xval_encoded = self.encode(Xval)
 model = self.build_model()
metrics = ClassificationReport(train_data=(Xtr_encoded,Ytr),val_data=(Xval_encoded,Yval))
model.fit(Xtr_encoded,Ytr,callbacks=metrics,batch_size=self.batch_size,epochs=self.epoch,\
validation_data=(Xval_encoded,Yval))
self.models.append(model)
 self.scores[fold] = {
'train':{
"precision" : metrics.train_precision_scores,
"recall":metrics.train_recall_scores,
"f1":metrics.train_f1_scores},
"validation":{
"precision":metrics.val_precision_scores,
"recall":metrics.val_precision_scores,
"f1":metrics.val_f1_scores
}
}
def plot_curve(self):
fig,axes = plt.subplots(nrows=k,ncols=2,figsize=(20,k*6),dpi=100)
 for i in range(k):
sns.lineplot(x=np.arange(1,self.epoch+1),y=self.models[i].history.history["val_accuracy"],\
ax=axes[i][0],label="val_accuracy")
 sns.lineplot(x=np.arange(1,self.epoch+1),y=self.scores[i]["validation"]["precision"],\
ax=axes[i][0],label="val_precision")
 sns.lineplot(x=np.arange(1,self.epoch+1),y=self.scores[i]["validation"]["recall"],\
ax=axes[i][0],label="val_recall")
 sns.lineplot(x=np.arange(1,self.epoch+1),y=self.scores[i]["validation"]["f1"],\
ax=axes[i][0],label="val_f1")
 axes[i][0].legend()
axes[i][0].set_title("Fold {} Validation classification Reports".format(i),fontsize=14)
 sns.lineplot(x=np.arange(1,self.epoch+1),y=self.models[i].history.history["loss"],\
ax=axes[i][1],label="train_loss")
 sns.lineplot(x=np.arange(1,self.epoch+1),y=self.models[i].history.history["val_loss"],\
ax =axes[i][1],label="val_loss")
 axes[i][1].legend()
axes[i][1].set_title("Fold {} Train / Validation Loss".format(i),fontsize=14)
 for j in range(2):
axes[i][j].set_xlabel('Epoch',size=12)
 axes[i][j].tick_params(axis="x",labelsize=12)
 axes[i][j].tick_params(axis="y",labelsize=12)
 plt.show()
def predict(self,X):
encoded_X = self.encode(X)
 prediction=[]
for i in range(k):
prediction.append(self.models[i].predict(encoded_X ).T)
 return np.mean(prediction,axis=0 ).T<train_model>
hp[categorical_oh_features] = hp[categorical_oh_features].astype('category' )
clf = BertDisasterDetecter(max_seq_length=128,lr=1e-3,epoch=2,batch_size=32)
 clf.train()<predict_on_test>
hp[categorical_l_features] = hp[categorical_l_features].astype('category' )
ypred= clf.predict(test_df["text_cleaned"].values )<save_to_csv>
hp[numerical_features] = hp[numerical_features].apply(pd.to_numeric, errors='coerce' )
model_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 model_submission['target'] = np.round(ypred ).astype('int')
 model_submission.to_csv('model_submission.csv', index=False)
 model_submission.describe()<load_from_csv>
hp.drop(columns=unused_features, inplace=True )
train_df= pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )<install_modules>
discrete_cols=[]
for col in hp.columns:
if hp[col].dtype=='int64' and len(hp[col].unique())<=15:
discrete_cols.append(col)
 discrete_cols
!pip install -U tensorflow_text==2.3 
<install_modules>
continuous_cols=[]
for col in hp.columns:
if(hp[col].dtype=='int' or hp[col].dtype=='float')and col not in ['Id', 'YearBuilt','YearRemodAdd','GarageYrBlt'] and col not in discrete_cols:
continuous_cols.append(col)
 continuous_cols
!pip install -q tf-models-official==2.3 
<import_modules>
t = abs(corr.SalePrice ).sort_values(ascending=False)
 print(t)
 t.index
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text as text
from official.nlp import optimization<create_dataframe>
cat1 = ['MSSubClass']
batch_size = 32
seed = 42
train_ds = tf.data.Dataset.from_tensor_slices(( train_df['text'].tolist() ,train_df['target'].tolist())).batch(batch_size )<define_variables>
pt = hp.pivot_table(index='MSSubClass', values='SalePrice', aggfunc=np.median )
bert_model_name = 'bert_en_uncased_L-12_H-768_A-12'
map_name_to_handle = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_base/2',
'electra_small':
'https://tfhub.dev/google/electra_small/2',
'electra_base':
'https://tfhub.dev/google/electra_base/2',
'experts_pubmed':
'https://tfhub.dev/google/experts/bert/pubmed/2',
'experts_wiki_books':
'https://tfhub.dev/google/experts/bert/wiki_books/2',
'talking-heads_base':
'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',
}
map_model_to_preprocess = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/2',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_preprocess/2',
'electra_small':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'electra_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'experts_pubmed':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'experts_wiki_books':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'talking-heads_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
}
tfhub_handle_encoder = map_name_to_handle[bert_model_name]
tfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]
print(f'BERT model selected : {tfhub_handle_encoder}')
 print(f'Preprocess model auto-selected: {tfhub_handle_preprocess}' )<categorify>
pt.sort_values(by='SalePrice',ascending=False )
def build_classifier_model() :
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string, name='text')
 preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation= "sigmoid" , name='classifier' )(net)
 return tf.keras.Model(text_input, net )<choose_model_class>
cat2=['MSZoning','Street','LotShape','LandContour','LotConfig','LandSlope']
loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
 metrics = tf.metrics.BinaryAccuracy()<choose_model_class>
year_features = ['YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']
epochs = 20
steps_per_epoch = tf.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * epochs
num_warmup_steps = int(0.1*num_train_steps)
 init_lr = 3e-5
optimizer = optimization.create_optimizer(init_lr=init_lr,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
optimizer_type='adamw' )<choose_model_class>
t = pd.DataFrame()
classifier_model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics )<train_model>
t['SalePrice'] = hp.SalePrice
t['YrSold'] = hp.YrSold
t['YearBuilt'] = hp.YrSold - hp.YearBuilt
t['YearRemodAdd'] = hp.YrSold - hp.YearRemodAdd
t['GarageYrBlt'] = hp.YrSold - hp.GarageYrBlt
print(f'Training model with {tfhub_handle_encoder}')
 history = classifier_model.fit(x=train_ds, epochs=epochs )<predict_on_test>
nulls = hp.isna().sum().sort_values(ascending=False)
 nulls_df = pd.DataFrame({'feature': nulls.index, 'value': nulls.values})
 nulls_df
probs = classifier_model.predict(test_df["text"])
 threshold = 0.4
preds = np.where(probs[:,] > threshold, 1, 0 )<load_from_csv>
nulls_with_type = pd.concat([hp[nulls_df.feature.values].dtypes, nulls], axis=1, keys=['type', 'count_of_nulls'])
 nulls_with_type[nulls_with_type.count_of_nulls > 0]
submission=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )<prepare_output>
numericals_nulls = nulls_with_type[nulls_with_type.index.isin(numerical_features)]
numericals_nulls
submission["target"]=preds<save_to_csv>
hp[numerical_features].isnull().sum()
submission.to_csv('submission.csv', index=False, header=True )<set_options>
numerical_imputer = impute.SimpleImputer(missing_values=np.nan, strategy='median')
 hp[numerical_features] = numerical_imputer.fit_transform(hp[numerical_features] )
pd.options.mode.chained_assignment = None
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 1000 )<load_from_csv>
hp[numerical_features].isnull().sum().sum()
train_origin_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_origin_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 train_df = train_origin_df.copy()
test_df = test_origin_df.copy()
def callback(operation_future):
result = operation_future.result()<concatenate>
date_nulls = nulls_with_type[nulls_with_type.index.isin(date_features)]
date_nulls
dataset_origin_df = pd.concat([train_df, test_df], axis=0, sort=True)
 dataset_origin_df.reset_index(inplace=True, drop=True )<create_dataframe>
date_nulls_above_zero = date_nulls[date_nulls.count_of_nulls > 0]
date_nulls_above_zero
words_containing_alpha_df = pd.DataFrame(columns = ['word', 'real', 'fake'])
 for row in tqdm(dataset_origin_df.iterrows()):
row = row[1]
result = re.findall('@+\w*', row['text'])
 if(len(result)> 0):
for word in result:
real = 1 if row['target'] == 1 else 0
fake = 1 if row['target'] != 1 else 0
temp_df = pd.DataFrame([[word, real, fake]], columns = ['word', 'real', 'fake'])
 words_containing_alpha_df = words_containing_alpha_df.append(temp_df )<sort_values>
date_imputer = impute.SimpleImputer(missing_values=np.nan, strategy='median')
 hp[date_features] = date_imputer.fit_transform(hp[date_features] )
words_containing_alpha_df = words_containing_alpha_df.groupby(['word'], as_index=False)['real', 'fake'].sum()
words_containing_alpha_df['total'] = words_containing_alpha_df['real'] + words_containing_alpha_df['fake']
words_containing_alpha_df = words_containing_alpha_df.sort_values(by = 'total', ascending = False)
 words_containing_alpha_df<filter>
hp[date_features].isnull().sum().sum()
threshold_no_of_words = 5
threshold_ratio_of_real_or_fake = 0.75
sub_words_df = words_containing_alpha_df.loc[(words_containing_alpha_df['total'] >= threshold_no_of_words)&(words_containing_alpha_df[['real', 'fake']].max(axis=1)/words_containing_alpha_df['total'] >= threshold_ratio_of_real_or_fake), :]
sub_words_df<drop_column>
categorical_features_to_impute_none = []
sub_words_df.drop(0, axis=0, inplace=True )<string_transform>
_features_to_drop = ['PoolQC','MiscFeature','Alley','Fence']
def remove_words_containing_alpha_by_threshold(text):
containing_words = re.findall('@+\w*', text)
 if(len(containing_words)> 0):
for word in containing_words:
if(word not in np.array(sub_words_df['word'])) :
text = text.replace(word, " ID")
 return text<categorify>
unused_features.extend(_features_to_drop )
def clean_1_2_1(tweet):
tweet = re.sub(r"%20", " ", tweet)
 tweet = re.sub(r"
", " ", tweet)
 tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 return tweet
def clean_1_2_2(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet
def clean_1_2_3(tweet):
tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 return tweet
def clean_1_2_4(tweet):
tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with ", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015/08/05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015/08/06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 return tweet
def clean_1_2_5(tweet):
tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 return tweet
def clean_1_2_6(tweet):
tweet = re.sub(r"\.{3,}", '...', tweet)
 if '...' not in tweet:
tweet = tweet.replace('.. ', '...')
 return tweet
def clean_1_2_7(tweet):
tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 tweet = re.sub(r"mh370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"legionnaires", "pneumonia", tweet)
 tweet = re.sub(r"derailment", "railway accident", tweet)
 tweet = re.sub(r"inundated", "flood", tweet)
 tweet = re.sub(r"deluged", "flood", tweet)
 tweet = re.sub(r"curfew", "stay at home", tweet)
 tweet = re.sub(r"obliteration", "destruction", tweet)
 tweet = re.sub(r"quarantine", "prevent the spread of disease", tweet)
 tweet = re.sub(r"lol", "laugh", tweet)
 tweet = re.sub(r"obliterate", "destroy", tweet)
 tweet = re.sub(r"hijacking", "seize", tweet)
 tweet = re.sub(r"detonation", "explosion", tweet)
 tweet = re.sub(r"electrocuted", "killed", tweet)
 tweet = re.sub(r"destroyd", "destroyed", tweet)
 return tweet
def clean_1_2_8(tweet):
tweet = re.sub(r"'cause", "because", tweet)
 tweet = re.sub(r"could've", "could have", tweet)
 tweet = re.sub(r"hadn't", "had not", tweet)
 tweet = re.sub(r"he'd", "he would", tweet)
 tweet = re.sub(r"how'd", "how did", tweet)
 tweet = re.sub(r"how'd'y", "how do you", tweet)
 tweet = re.sub(r"how'll", "how will", tweet)
 tweet = re.sub(r"how's", "how is", tweet)
 tweet = re.sub(r"I'd've", "I would have", tweet)
 tweet = re.sub(r"I'll've", "I will have", tweet)
 tweet = re.sub(r"i'd've", "i would have", tweet)
 tweet = re.sub(r"i'll've", "i will have", tweet)
 tweet = re.sub(r"it'd", "it would", tweet)
 tweet = re.sub(r"it'd've", "it would have", tweet)
 tweet = re.sub(r"it'll've", "it will have", tweet)
 tweet = re.sub(r"ma'am", "madam", tweet)
 tweet = re.sub(r"mayn't", "may not", tweet)
 tweet = re.sub(r"might've", "might have", tweet)
 tweet = re.sub(r"mightn't", "might not", tweet)
 tweet = re.sub(r"mightn't've", "might not have", tweet)
 tweet = re.sub(r"must've", "must have", tweet)
 tweet = re.sub(r"mustn't", "must not", tweet)
 tweet = re.sub(r"mustn't've", "must not have", tweet)
 tweet = re.sub(r"needn't", "need not", tweet)
 tweet = re.sub(r"needn't've", "need not have", tweet)
 tweet = re.sub(r"o'clock", "of the clock", tweet)
 tweet = re.sub(r"oughtn't", "ought not", tweet)
 tweet = re.sub(r"oughtn't've", "ought not have", tweet)
 tweet = re.sub(r"shan't", "shall not", tweet)
 tweet = re.sub(r"sha'n't", "shall not", tweet)
 tweet = re.sub(r"shan't've", "shall not have", tweet)
 tweet = re.sub(r"she'd", "she would", tweet)
 tweet = re.sub(r"she'd've", "she would have", tweet)
 tweet = re.sub(r"she'll", "she will", tweet)
 tweet = re.sub(r"she'll've", "she will have", tweet)
 tweet = re.sub(r"she's", "she is", tweet)
 tweet = re.sub(r"shouldn't've", "should not have", tweet)
 tweet = re.sub(r"so've", "so have", tweet)
 tweet = re.sub(r"so's", "so as", tweet)
 tweet = re.sub(r"this's", "this is", tweet)
 tweet = re.sub(r"that'd", "that would", tweet)
 tweet = re.sub(r"that'd've", "that would have", tweet)
 tweet = re.sub(r"there'd", "there would", tweet)
 tweet = re.sub(r"there'd've", "there would have", tweet)
 tweet = re.sub(r"they'd've", "they would have", tweet)
 tweet = re.sub(r"they'll've", "they will have", tweet)
 tweet = re.sub(r"to've", "to have", tweet)
 tweet = re.sub(r"we'd've", "we would have", tweet)
 tweet = re.sub(r"we'll've", "we will have", tweet)
 tweet = re.sub(r"what'll", "what will", tweet)
 tweet = re.sub(r"what'll've", "what will have", tweet)
 tweet = re.sub(r"what're", "what are", tweet)
 tweet = re.sub(r"what've", "what have", tweet)
 tweet = re.sub(r"when's", "when is", tweet)
 tweet = re.sub(r"when've", "when have", tweet)
 tweet = re.sub(r"where'd", "where did", tweet)
 tweet = re.sub(r"where've", "where have", tweet)
 tweet = re.sub(r"who'll", "who will", tweet)
 tweet = re.sub(r"who'll've", "who will have", tweet)
 tweet = re.sub(r"who've", "who have", tweet)
 tweet = re.sub(r"why's", "why is", tweet)
 tweet = re.sub(r"why've", "why have", tweet)
 tweet = re.sub(r"will've", "will have", tweet)
 tweet = re.sub(r"won't've", "will not have", tweet)
 tweet = re.sub(r"wouldn't've", "would not have", tweet)
 tweet = re.sub(r"y'all'd", "you all would", tweet)
 tweet = re.sub(r"y'all'd've", "you all would have", tweet)
 tweet = re.sub(r"y'all're", "you all are", tweet)
 tweet = re.sub(r"y'all've", "you all have", tweet)
 tweet = re.sub(r"you'd've", "you would have", tweet)
 tweet = re.sub(r"you'll've", "you will have", tweet)
 return tweet<categorify>
categorical_oh_features = list(set(categorical_oh_features ).difference(set(_features_to_drop)) )
def remove_url(text):
url_pattern = re.compile(r'https?://\S*|www\.\S*')
 return url_pattern.sub(r'URL', text)
 def remove_url_for_labelling(text):
url_pattern = re.compile(r'https?://\S*|www\.\S*')
 return url_pattern.sub(r'', text )<define_variables>
categorical_l_features = list(set(categorical_l_features ).difference(set(_features_to_drop)) )
EMOTICONS = {
u":‑\)":"Happy face smiley",
u":\)":"Happy face smiley",
u":-\]":"Happy face smiley",
u":\]":"Happy face smiley",
u":-3":"Happy face smiley",
u":->":"Happy face smiley",
u":>":"Happy face smiley",
u"8-\)":"Happy face smiley",
u":o\)":"Happy face smiley",
u":-\}":"Happy face smiley",
u":\}":"Happy face smiley",
u":-\)":"Happy face smiley",
u":c\)":"Happy face smiley",
u":\^\)":"Happy face smiley",
u"=\]":"Happy face smiley",
u"=\)":"Happy face smiley",
u":‑D":"Laughing, big grin laugh with glasses",
u"8‑D":"Laughing, big grin laugh with glasses",
u"X‑D":"Laughing, big grin laugh with glasses",
u"B\^D":"Laughing, big grin laugh with glasses",
u":-\)\)":"Very happy",
u":‑\(":"Frown, sad, andry pouting",
u":-\(":"Frown, sad, andry pouting",
u":\(":"Frown, sad, andry pouting",
u":‑c":"Frown, sad, andry pouting",
u":‑<":"Frown, sad, andry pouting",
u":<":"Frown, sad, andry pouting",
u":‑\[":"Frown, sad, andry pouting",
u":\[":"Frown, sad, andry pouting",
u":-\|\|":"Frown, sad, andry pouting",
u">:\[":"Frown, sad, andry pouting",
u":\{":"Frown, sad, andry pouting",
u":@":"Frown, sad, andry pouting",
u">:\(":"Frown, sad, andry pouting",
u":'‑\(":"Crying",
u":'\(":"Crying",
u":'‑\)":"Tears of happiness",
u":'\)":"Tears of happiness",
u"D:<":"Disgust",
u":‑O":"Surprise",
u":‑o":"Surprise",
u":-0":"Shock",
u"8‑0":"Yawn",
u">:O":"Yawn",
u":-\*":"Kiss",
u":\*":"Kiss",
u";‑\)":"Wink smirk",
u";\)":"Wink smirk",
u"\*-\)":"Wink smirk",
u"\*\)":"Wink smirk",
u";‑\]":"Wink smirk",
u";\]":"Wink smirk",
u";\^\)":"Wink smirk",
u":‑,":"Wink smirk",
u":‑P":"Tongue sticking out, cheeky, playful blowing a raspberry",
u"X‑P":"Tongue sticking out, cheeky, playful blowing a raspberry",
u":‑Þ":"Tongue sticking out, cheeky, playful blowing a raspberry",
u">:P":"Tongue sticking out, cheeky, playful blowing a raspberry",
u":‑/":"Skeptical, annoyed, undecided, uneasy hesitant",
u":/":"Skeptical, annoyed, undecided, uneasy hesitant",
u":-[.]":"Skeptical, annoyed, undecided, uneasy hesitant",
u">:[(\\\)]":"Skeptical, annoyed, undecided, uneasy hesitant",
u">:/":"Skeptical, annoyed, undecided, uneasy hesitant",
u":[(\\\)]":"Skeptical, annoyed, undecided, uneasy hesitant",
u"=/":"Skeptical, annoyed, undecided, uneasy hesitant",
u"=[(\\\)]":"Skeptical, annoyed, undecided, uneasy hesitant",
u":‑\|":"Straight face",
u":\|":"Straight face",
u":\$":"Embarrassed blushing",
u":‑x":"Sealed lips wearing braces tongue-tied",
u":‑
u":‑&":"Sealed lips wearing braces tongue-tied",
u":&":"Sealed lips wearing braces tongue-tied",
u"O:‑\)":"Angel, saint innocent",
u"O:\)":"Angel, saint innocent",
u"0:‑3":"Angel, saint innocent",
u"0:‑\)":"Angel, saint innocent",
u"0:\)":"Angel, saint innocent",
u":‑b":"Tongue sticking out, cheeky, playful blowing a raspberry",
u"0;\^\)":"Angel, saint innocent",
u">:‑\)":"Evil devilish",
u">:\)":"Evil devilish",
u"\}:‑\)":"Evil devilish",
u"\}:\)":"Evil devilish",
u"3:‑\)":"Evil devilish",
u"3:\)":"Evil devilish",
u">;\)":"Evil devilish",
u"\|;‑\)":"Cool",
u"\|‑O":"Bored",
u":‑J":"Tongue-in-cheek",
u"
u"%‑\)":"Drunk confused",
u":-
u":
u"<:‑\|":"Dump",
u"\(>_<\)":"Troubled",
u"\(>_<\)>":"Troubled",
u"\(';'\)":"Baby",
u"\(\^\^>``":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(\^_\^;\)":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(-_-;\)":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(~_~;\)\(・\.・;\)":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(-_-\)zzz":"Sleeping",
u"\(\^_-\)":"Wink",
u"\(\(\+_\+\)\)":"Confused",
u"\(\+o\+\)":"Confused",
u"\(o\|o\)":"Ultraman",
u"\^_\^":"Joyful",
u"\(\^_\^\)/":"Joyful",
u"\(\^O\^\)／":"Joyful",
u"\(\^o\^\)／":"Joyful",
u"\(__\)":"Kowtow as a sign of respect, dogeza for apology",
u"_\(\._\.\)_":"Kowtow as a sign of respect, dogeza for apology",
u"<\(_ _\)>":"Kowtow as a sign of respect, dogeza for apology",
u"<m\(__\)m>":"Kowtow as a sign of respect, dogeza for apology",
u"m\(__\)m":"Kowtow as a sign of respect, dogeza for apology",
u"m\(_ _\)m":"Kowtow as a sign of respect, dogeza for apology",
u"\('_'\)":"Sad Crying",
u"\(/_;\)":"Sad Crying",
u"\(T_T\)\(;_;\)":"Sad Crying",
u"\(;_;":"Sad of Crying",
u"\(;_:\)":"Sad Crying",
u"\(;O;\)":"Sad Crying",
u"\(:_;\)":"Sad Crying",
u"\(ToT\)":"Sad Crying",
u";_;":"Sad Crying",
u";-;":"Sad Crying",
u";n;":"Sad Crying",
u"Q\.Q":"Sad Crying",
u"T\.T":"Sad Crying",
u"Q_Q":"Sad Crying",
u"\(-\.-\)":"Shame",
u"\(-_-\)":"Shame",
u"\(一一\)":"Shame",
u"\(；一_一\)":"Shame",
u"\(=_=\)":"Tired",
u"\(=\^\·\^=\)":"cat",
u"\(=\^\·\·\^=\)":"cat",
u"=_\^=	":"cat",
u"\(\.\.\)":"Looking down",
u"\(\._\.\)":"Looking down",
u"\^m\^":"Giggling with hand covering mouth",
u">\^_\^<":"Normal Laugh",
u"<\^!\^>":"Normal Laugh",
u"\^/\^":"Normal Laugh",
u"\（\*\^_\^\*）" :"Normal Laugh",
u"\(\^<\^\)\(\^\.\^\)":"Normal Laugh",
u"\(^\^\)":"Normal Laugh",
u"\(\^\.\^\)":"Normal Laugh",
u"\(\^_\^\.\)":"Normal Laugh",
u"\(\^_\^\)":"Normal Laugh",
u"\(\^\^\)":"Normal Laugh",
u"\(\^J\^\)":"Normal Laugh",
u"\(\*\^\.\^\*\)":"Normal Laugh",
u"\(\^—\^\）":"Normal Laugh",
u"\(
 u"\（\^—\^\）":"Waving",
u"\(;_;\)/~~~":"Waving",
u"\(\^\.\^\)/~~~":"Waving",
u"\(-_-\)/~~~ \($\·\·\)/~~~":"Waving",
u"\(T_T\)/~~~":"Waving",
u"\(ToT\)/~~~":"Waving",
u"\(\*\^0\^\*\)":"Excited",
u"\(\*_\*\)":"Amazed",
u"\(\*_\*;":"Amazed",
u"\(\+_\+\)\(@_@\)":"Amazed",
u"\(\*\^\^\)v":"Laughing,Cheerful",
u"\(\^_\^\)v":"Laughing,Cheerful",
u"\(\(d[-_-]b\)\)":"Headphones,Listening to music",
u'\(-"-\)':"Worried",
u"\(ーー;\)":"Worried",
u"\(\^0_0\^\)":"Eyeglasses",
u"\(\＾ｖ\＾\)":"Happy",
u"\(\＾ｕ\＾\)":"Happy",
u"\(\^\)o\(\^\)":"Happy",
u"\(\^O\^\)":"Happy",
u"\(\^o\^\)":"Happy",
u"\)\^o\^\(":"Happy",
u":O o_O":"Surprised",
u"o_0":"Surprised",
u"o\.O":"Surpised",
u"\(o\.o\)":"Surprised",
u"\(\*￣m￣\)":"Dissatisfied",
u"\(‘A`\)":"Snubbed Deflated",
u':)':"Standard smile",
u':-)':"With nose",
u':-E':"Buck-tooth Vampire",
u'>-)':"Evil grin",
u':(':"Sad frown smile",
u':-(':"Sad with nose",
u':-<':"Super sad",
u':-O':"Surprised",
u':-*':"Kissing",
u':-@':"Angry",
u':-$':"Confused",
u':-'':"Shifty",
u':-
u'(((H)))':"Hugs",
u':-X':"Kiss on the lips",
u'`:-)':"One eyebrow raised",
u':^)':"A broken nose",
u':-&':"tongue tied",
u'E-:-)':"a Ham radio operator",
u'<:-)':"Uni-brow",
u':->':"Big grin happy",
u'(-}{-)':"Couple kissing",
u':-Q':"Smoking",
u'$_$':"Greedy",
u':-!': ,
u':-D':"Laughter",
u':*)':"Drunk smile",
u':@':"Exclamation ""What???<define_variables>
hp.drop(columns=_features_to_drop,axis=1,inplace=True )
EMOTICONS_fix = {
u":‑\)":"happy",
u":\)":"happy",
u":-\]":"happy",
u":\]":"happy",
u":-3":"happy",
u":->":"happy",
u":>":"happy",
u"8-\)":"happy",
u":o\)":"happy",
u":-\}":"happy",
u":\}":"happy",
u":-\)":"happy",
u":c\)":"happy",
u":\^\)":"happy",
u"=\]":"happy",
u"=\)":"happy",
u":‑D":"happy",
u"8‑D":"happy",
u"X‑D":"happy",
u"B\^D":"happy",
u":-\)\)":"happy",
u":‑\(":"sad",
u":-\(":"sad",
u":\(":"sad",
u":‑c":"sad",
u":‑<":"sad",
u":<":"sad",
u":‑\[":"sad",
u":\[":"sad",
u":-\|\|":"sad",
u">:\[":"sad",
u":\{":"sad",
u":@":"sad",
u">:\(":"sad",
u":'‑\(":"sad",
u":'\(":"sad",
u":'‑\)":"happy",
u":'\)":"happy",
u"D:<":"Disgust",
u":‑O":"surprise",
u":‑o":"surprise",
u":-0":"shocked",
u"8‑0":"Yawn",
u">:O":"Yawn",
u":-\*":"Kiss you",
u":\*":"Kiss you",
u":‑P":"giggle",
u"X‑P":"giggle",
u":‑Þ":"giggle",
u">:P":"giggle",
u":‑/":"sad",
u":/":"sad",
u":-[.]":"sad",
u">:[(\\\)]":"sad",
u">:/":"sad",
u":[(\\\)]":"sad",
u"=/":"sad",
u"=[(\\\)]":"sad",
u":\$":"Embarrassed",
u":‑x":"silent",
u":‑
u":‑&":"silent",
u":&":"silent",
u":‑b":"giggle",
u"\|;‑\)":"Cool",
u"\|‑O":"Bored",
u":-
u":
u"<:‑\|":"Dump",
u"\(>_<\)":"Troubled",
u"\(>_<\)>":"Troubled",
u"\(';'\)":"Baby",
u"\(\^\^>``":"Nervous",
u"\(\^_\^;\)":"Nervous",
u"\(-_-;\)":"Nervous",
u"\(~_~;\)\(・\.・;\)":"Nervous",
u"\(-_-\)zzz":"Sleeping",
u"\(\^_-\)":"Wink",
u"\(\(\+_\+\)\)":"Confused",
u"\(\+o\+\)":"Confused",
u"\(o\|o\)":"Ultraman",
u"\^_\^":"Joyful",
u"\(\^_\^\)/":"Joyful",
u"\(\^O\^\)／":"Joyful",
u"\(\^o\^\)／":"Joyful",
u"\(__\)":"apology",
u"_\(\._\.\)_":"apology",
u"<\(_ _\)>":"apology",
u"<m\(__\)m>":"apology",
u"m\(__\)m":"apology",
u"m\(_ _\)m":"apology",
u"\('_'\)":"sad",
u"\(/_;\)":"sad",
u"\(T_T\)\(;_;\)":"sad",
u"\(;_;":"sad",
u"\(;_:\)":"sad",
u"\(;O;\)":"sad",
u"\(:_;\)":"sad",
u"\(ToT\)":"sad",
u";_;":"sad",
u";-;":"sad",
u";n;":"sad",
u"Q\.Q":"sad",
u"T\.T":"sad",
u"Q_Q":"sad",
u"\(-\.-\)":"Shame",
u"\(-_-\)":"Shame",
u"\(一一\)":"Shame",
u"\(；一_一\)":"Shame",
u"\(=_=\)":"Tired",
u"\(\.\.\)":"Looking down",
u"\(\._\.\)":"Looking down",
u"\^m\^":"Giggle",
u">\^_\^<":"Laugh",
u"<\^!\^>":"Laugh",
u"\^/\^":"Laugh",
u"\（\*\^_\^\*）" :"Laugh",
u"\(\^<\^\)\(\^\.\^\)":"Laugh",
u"\(^\^\)":"Laugh",
u"\(\^\.\^\)":"Laugh",
u"\(\^_\^\.\)":"Laugh",
u"\(\^_\^\)":"Laugh",
u"\(\^\^\)":"Laugh",
u"\(\^J\^\)":"Laugh",
u"\(\*\^\.\^\*\)":"Laugh",
u"\(\^—\^\）":"Laugh",
u"\(
 u"\（\^—\^\）":"Waving",
u"\(;_;\)/~~~":"Waving",
u"\(\^\.\^\)/~~~":"Waving",
u"\(-_-\)/~~~ \($\·\·\)/~~~":"Waving",
u"\(T_T\)/~~~":"Waving",
u"\(ToT\)/~~~":"Waving",
u"\(\*\^0\^\*\)":"Excited",
u"\(\*_\*\)":"Amazed",
u"\(\*_\*;":"Amazed",
u"\(\+_\+\)\(@_@\)":"Amazed",
u"\(\*\^\^\)v":"Laugh",
u"\(\^_\^\)v":"Laugh",
u"\(\(d[-_-]b\)\)":"Listening to music",
u'\(-"-\)':"Worried",
u"\(ーー;\)":"Worried",
u"\(\^0_0\^\)":"Eyeglasses",
u"\(\＾ｖ\＾\)":"happy",
u"\(\＾ｕ\＾\)":"happy",
u"\(\^\)o\(\^\)":"happy",
u"\(\^O\^\)":"happy",
u"\(\^o\^\)":"happy",
u"\)\^o\^\(":"happy",
u":O o_O":"Surprised",
u"o_0":"Surprised",
u"o\.O":"Surpised",
u"\(o\.o\)":"Surprised",
u"\(\*￣m￣\)":"Dissatisfied",
u"\(‘A`\)":"Snubbed Deflated",
u':)':"Laugh",
u':-)':"Laugh",
u':(':"sad",
u':-(':"sad",
u':-<':"sad",
u':-O':"Surprised",
u':-*':"Kissing",
u':-@':"Angry",
u':-$':"Confused",
u':-'':"Shifty",
u":-
}<categorify>
categorical_oh_nulls = nulls_with_type[nulls_with_type.index.isin(categorical_oh_features)]
categorical_oh_nulls
def convert_emoticons(text):
for emot in EMOTICONS_fix.items() :
if emot[0] in text:
text = text.replace(emot[0],".I feel "+emot[1]+".")
 return text<drop_column>
categorical_l_nulls = nulls_with_type[nulls_with_type.index.isin(categorical_l_features)]
categorical_l_nulls
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r' ', text)
 remove_emoji("Omg another Earthquake 😔😔" )<define_variables>
categorical_features_to_impute_none += ['FireplaceQu']
Abbreviation = {
'?':"I have a question",
'?4U':"I have a question for you",
';S':"Hmm? What did you say?",
'^^':"read line",
'<3':"sideways heart",
'<3':"broken heart",
'<33':"heart or love",
'@TEOTD':"At the end of the day",
'0.02':"My(or your)two cents worth",
"'1TG, 2TG'":" number of items needed for win ",
'1UP':"extra life",
'121':"One-to-one",
'1337':"Leet, 'elite'",
'143':"I love you",
'1432':"I love you too",
'14AA41':"One for all, and all for one",
'182':"I hate you",
'19':"Zero hand",
'10M':"Ten man",
'10X':"Thanks",
'10Q':"Thank you",
'1CE':"Once",
'1DR':"I wonder",
'1NAM':"One in a million",
'2': "to"" in SMS",
'20': "location",
'2B':"To be",
'2EZ':"Too easy",
'2G2BT':"Too good to be true",
'2M2H':"Too much too handle",
'2MI':"Too much information",
'2MOR':"Tomorrow",
'2MORO':"Tomorrow",
'2M2H':"Too much to handle",
'2N8':"Tonight",
'2NTE':"Tonight",
'4':'for',
'411': "information",
'404':"I don't know",
'411':'information',
'420':"Lets get high",
'420': "Marijuana",
'459':"I love you",
'4AO':"For adults only",
'4COL':"For crying out loud",
'4EAE':"Forever and ever",
'4EVA':"Forever",
'4NR':"Foreigner",
'4SALE':"For sale",
'^5':"High-five",
'511':"Too much information",
'555':"Sobbing, crying.",
'55555':"Crying your eyes out",
'55555':"Laughing",
'6Y':"Sexy",
'7K':"Sick",
'81':"Hells Angels",
'831':"I love you",
'86':"Over",
'88':"Bye-bye",
'88':"Hugs and kisses",
'9':"Parent is watching ",
'*s*': "smile",
'*w*': "wink",
'A3':"Anytime, anywhere, anyplace",
'AA':"Alcoholics Anonymous",
'AA':"As above",
'AA':"Ask about",
'AAF':"As a matter of fact",
'AAF':"As a friend",
'AAK':"Asleep at keyboard",
'AAK':"Alive and kicking",
'AAMOF':"As a matter of fact",
'AAMOI':"As a matter of interest",
'AAP':"Always a pleasure",
'AAR':"At any rate",
'AAS':"Alive and smiling",
'AASHTA':"As always, Sheldon has the answer",
'AATK':"Always at the keyboard",
'AAYF':"As always, your friend",
'ABBR':"Meaning abbreviation",
'ABC':"Already been chewed",
'ABD':"Already been done",
'ABT':"About",
'ABT2':"About to",
'ABTA':"Good-bye",
'ABU':"All bugged up",
'AC':"Acceptable content",
'ACC':"Anyone can come",
'ACD':"ALT / CONTROL / DELETE",
'ACDNT':"Accident",
'ACE':"marijuana cigarette",
'ACK':"Acknowledge",
'ACPT':"Accept",
'ACQSTN':"Acquisition",
'ADAD':"Another day, another dollar",
'ADBB':"All done, bye-bye",
'ADD':"Address",
'ADDY':"Address",
'ADIH':"Another day in hell",
'ADIP':"Another day in paradise",
'ADMIN':"Administrator",
'ADMINR':"Administrator(Government)",
'ADN':"Any day now",
'ADR':"Address",
'AE':"Area effect ",
'AEAP':"As early as possible",
'AF':"April Fools",
'AF':"As *Freak*",
'AF':"Aggression factor",
'AFC':"Away from computer",
'AFAIAA':"As far as I am aware",
'AFAIC':"As far as I am concerned",
'AFAIK':"As far as I know",
'AFAIUI':"As far as I understand it",
'AFAP':"As far as possible",
'AFFA':"Angels Forever, Forever Angels",
'AFJ':"April Fool's joke",
'AFK':"Away from keyboard",
'AFZ':"Acronym Free Zone",
'AFPOE':"A fresh pair of eyes",
'AGI': "agility"" ",
'AH':"At home",
'AIAMU':"And I am a money's uncle",
'AIGHT':"Alright",
'AIR':"As I remember",
'AISB':"As it should be",
'AISB':"As I said before",
'AISI':"As I see it",
'AITR':"Adult in the room",
'AKA':"Also known as",
'ALCON':"All concerned",
'ALOL':"Actually laughing out loud",
'AMA':"Ask me anything(Reddit)",
'AMAP':"As much as possible",
'AMBW':"All my best wishes",
'AML':"All my love",
'AMOF':"As a matter of fact",
'A/N':"Author's note",
'AO':"Anarchy Online ",
'AOC':"Available on cell",
'AOE':"Area of effect(online game)",
'AOM':"Age of majority",
'AOM':"Age of Mythology ",
'AOTA':"All of the above",
'AOYP':"Angel on your pillow",
'APAC':"All praise and credit",
'APP':"Application",
'APP':"Appreciate",
'AQAP':"As quick(or quiet)as possible",
'ARC':"Archive(compressed files)",
'ARE':"Acronym rich environment",
'ARG':"Argument",
'ASIG':"And so it goes",
'ASAP':"As soon as possible",
'A/S/L':"Age/sex/location",
'ASL':"Age/sex/location",
'ASLA':"Age/sex/location/availability",
'AT':"At your terminal",
'ATB':"All the best",
'ATEOTD':"At the end of the day",
'ATM':"At the moment",
'ATSITS':"All the stars in the sky",
'ATSL':"Along the same line(or lines)",
'AWC':"After awhile crocodile",
'AWESO':"Awesome",
'AWOL':"Away without leaving",
'AWOL':"Absent without leave",
'AYDY':"Are you done yet?",
'AYBABTU':"All your base are belong to us ",
'AYEC':"At your earliest convenience",
'AYOR':"At your own risk",
'AYSOS':"Are you stupid or something?",
'AYS':"Are you serious?",
'AYT':"Are you there?",
'AYTMTB':"And you're telling me this because",
'AYV':"Are you vertical?",
'AYW':"As you were",
'AYW':"As you want / As you wish",
'AZN':"Asian",
'B':"Back",
'B':"Be",
'B&':"Banned",
'B2B':"Business-to-business",
'B2C':"Business-to-consumer",
'B2W':"Back to work",
'B8':"Bait(person teased or joked with, or under-aged person/teen)",
'B9':"Boss is watching",
'B/F':"Boyfriend",
'B/G':"Background(personal information request)",
'B4':"Before",
'B4N':"Bye for now",
'BAG':"Busting a gut",
'BA':"Bad *a*",
'BAE':"Before anyone else",
'BAE':"Meaning Babe or baby",
'BAFO':"Best and final offer",
'BAK':"Back at keyboard",
'BAM':"Below average mentality",
'BAMF':"Bad *a* mother *f*",
'BAO':"Be aware of",
'BAS':"Big 'butt' smile",
'BASIC':"Meaning anything mainstream",
'BASOR':"Breathing a sigh of relief",
'BAU':"Business as usual",
'BAY':"Back at ya",
'BB':"Be back",
'BB':"Big brother",
'BB':"Bebi / Baby(Spanish SMS)",
'BBC':"Big bad challenge",
'BBIAB':"Be back in a bit",
'BBIAF':"Be back in a few",
'BBIAM':"Be back in a minute",
'BBIAS':"Be back in a sec",
'BBL':"Be back later",
'BBN':"Bye, bye now",
'BBQ':"Barbeque, ""Ownage"", shooting score/frag ",
'BBS':"Be back soon",
'BBT':"Be back tomorrow",
'BC':"Because",
'B/C':"Because",
'BC':"Be cool",
'BCNU':"Be seeing you",
'BCOS':"Because",
'BCO':"Big crush on",
'BCOY':"Big crush on you",
'BD':"Big deal",
'BDAY':"Birthday",
'B-DAY':"Birthday",
'BDN':"Big darn number",
'BEG':"Big evil grin",
'BELF': "Blood Elf"" ",
'BF':"Boyfriend",
'BF':"Brain fart",
'BFAW':"Best friend at work",
'BF2':"Battlefield 2 ",
'BF':"Best friend",
'BFF':"Best friends forever",
'BFFL':"Best friends for life",
'BFFLNMW':"Best friends for life, no matter what",
'BFD':"Big freaking deal",
'BFG':"Big freaking grin",
'BFFN':"Best friend for now",
'BFN':"Bye for now",
'BG':"Big grin",
'BGWM':"Be gentle with me",
'BHL8':"Be home late",
'BIB':"Boss is back",
'BIBO':"Beer in, beer out",
'BIC':"Butt in chair",
'BIF':"Before I forget",
'BIH':"Burn in hell",
'BIL':"Brother in law",
'BIO': "I'm going to the bathroom""(or)""Bathroom break",
'BION':"Believe it or not",
'BIOYA':"Blow it out your *a*",
'BIOYN':"Blow it out your nose",
'BIS':"Best in slot ",
'BISFLATM':"Boy, I sure feel like a turquoise monkey!(unverified)",
'BITMT':"But in the meantime",
'BL':"Belly laugh",
'BLNT':"Better luck next time",
'Bloke':"Meaning Man(Discord)",
'BM':"Bite me",
'BME':"Based on my experience",
'BM&Y':"Between me and you",
'BOB':"Back off *buddy*",
'BN':"Bad news",
'BOE': "bind on equip"" ",
'BOHICA':"Bend over here it comes again",
'BOL':"Best of luck",
'BOM':"*b* of mine",
'BOLO':"Be on the look out",
'BOOMS':"Bored out of my skull",
'BOP': "bind on pickup"" ",
'BOSMKL':"Bending over smacking my knee laughing",
'BOT':"Back on topic",
'BOT':"Be on that",
'BMS':"Broke my scale, used when rating someone",
'BOYF':"Boyfriend",
'BPLM':"Big person little mind",
'BRB':"Be right back",
'BR':"Best regards",
'BRBB':"Be right back *b*",
'BRNC':"Be right back, nature calls",
'BRD':"Bored",
'BRH':"Be right here",
'BRT':"Be right there",
'BSF':"But seriously folks",
'BSOD':"Blue screen of death",
'BSTS':"Better safe than sorry",
'BT':"Bite this",
'BT':"Between technologies",
'BTA':"But then again",
'BTDT':"Been there, done that",
'BTW':"By the way ",
'BTYCL':"Bootycall",
'BUBU':"Slang term for the most beautiful of women",
'BURN':"Used to reference an insult",
'Buff': "changed and is now stronger",
'BWL':"Bursting with laughter ",
'BYOB':"Bring your own beer ",
'BYOC':"Bring your own computer",
'BYOD':"Bring your own device",
'BYOH':"Bat you on head",
'BYOP':"Bring your own paint",
'BYTM':"Better you than me",
'C&G':"Chuckle & grin",
'C4N':"Ciao for now",
'CAD':"Control + Alt + Delete",
'CAD':"Short for Canada/Canadian",
'Cakeday':"Birthday",
'CAM':"Camera(SMS)",
'CB':"Coffee break",
'CB':"Chat break",
'CB':"Crazy *b*",
'CD9':"parents are around",
'CFS':"Care for secret?",
'CFY':"Calling for you",
'CHK':"Check",
'CIAO':"Good-bye",
'CICO':"Coffee in, coffee out",
'CID':"Crying in disgrace",
'CID':"Consider it done",
'CLAB':"Crying like a baby",
'CLD':"Could",
'CLK':"Click",
'CM':"Call me",
'CMAP':"Cover my *a* partner ",
'CMB':"Call me back",
'CMGR': "Community manager",
'CMIIW':"Correct me if I'm wrong",
'CMON':"Come on",
'CNP':"Continued(in)next post",
'COB':"Close of business",
'COH':"City of Heroes ",
'COS':"Because",
'C/P':"Cross post",
'CP':"Chat post",
'CR8':"Create",
'Cray':"Meaning crazy",
'CRE8':"Create",
'CRA CRA':"crazy",
'CRAFT':"Can't remember a *freaking* thing",
'CRB':"Come right back",
'CRBT ':"Crying really big tears ",
'CRIT ': "critical hit",
'CRZ ':"Crazy",
'CRS ':"Can't remember *stuff*",
'CSG':"Chuckle, snicker, grin",
'CSL':"Can't stop laughing",
'CSS':"Counter-Strike Source ",
'CT':"Can't talk",
'CTC':"Care to chat?",
'CTHU':"Cracking the *heck* up",
'CTN':"Can't talk now",
'CTO':"Check this out",
'CU':"See you too",
'CU':"See you",
'CU2':"See you",
'CUA':"See you around",
'CUL':"See you later",
'CULA':"See you later alligator",
'CUL8R':"See you later",
'CUMID':"See you in my dreams",
'CURLO':"See you around like a donut",
'CWD':"Comment when done",
'CWOT':"Complete waste of time",
'CWYL':"Chat with you later",
'CX': "Correction",
'CYA':"See you",
'CYAL8R':"See you later",
'CYE':"Check your e-mail",
'CYEP':"Close your eyes partner ",
'CYO':"See you online",
'D2':"Dedos",
'D46?':"Down for sex?",
'DA': "The",
'DAE':"Does anyone else?",
'DAFUQ':"(What)the *Freak*?",
'DAM':"Don't annoy me",
'DAoC':"Dark Age of Camelot ",
'DBAU':"Doing business as usual",
'DBEYR':"Don't believe everything you read",
'DC':"Disconnect",
'DD':"Dear daughter",
'DD':"Due diligence",
'DDG':"Drop dead gorgeous",
'DEEZ NUTZ':"A phrase used in online chat to annoy or frustrate those involved in the conversation.",
'DEGT':"Dear daughter",
'DERP':"stupid",
'DF':"Don't even go there",
'DFL':"Dead *freaking* last ",
'DGA':"Don't go anywhere",
'DGAF':"Don't give a *freak*",
'DGT':"Don't go there",
'DGTG':"Don't go there, girlfriend",
'DGYF':"Dang, girl you fine",
'DH':"Dear husband",
'DHU':"Dinosaur hugs(used to show support)",
'DIIK':"Darned if I know",
'DIKU':"Do I know you?",
'DILLIGAF':"Do I look like I give a *freak*?",
'DILLIGAS':"Do I look like I give a sugar?",
'DIS':"Did I say?",
'DITYID':"Did I tell you I'm distressed?",
'DIY':"Do it yourself",
'DKDC':"Don't know, don't care",
'DKP':"Dragon kill points ",
'D/L':"Download",
'DL':"Download",
'DL':"Down low",
'DL':"Dead link",
'DLBBB':"Don't let(the)bed bugs bite",
'DLTBBB':"Don't let the bed bugs bite",
'DM':"Doesn't matter",
'DM':"Direct message",
'DM':"Do me",
'DM':"Dungeon Master ",
'DMNO':"Dude Man No Offense",
'DMY':"Don't mess yourself",
'DN':"Down",
'DNC':"I do not understand",
'DNR':"Dinner",
'DNT':"Don't",
'd00d':"Dude",
'DOE':"Daughter of Eve",
'DORBS': "Adorable",
'DOT':"Damage over time ",
'Downvote':"Voting negatively on a thread using Reddit’s voting system",
'DPS':"Damage per second ",
'DQMOT':"Don't quote me on this",
'DR':"Didn't read",
'DS':"Dear son",
'DTR':"Define the relationship",
'DTRT':"Do the right thing",
'DTS':"Don't think so",
'DTTD':"Don't touch that dial",
'DUPE':"Duplicate",
'DUR':"Do you remember?",
'DV8':"Deviate",
'DW':"Dear wife",
'DWF':"Divorced white female",
'DWM':"Divorced white male",
'DXNRY':"Dictionary",
'DYNWUTB':"Do you know what you are talking about?",
'DYFI':"Did you find it?",
'DYFM':"Dude, you fascinate me",
'DYJHIW':"Don't you just hate it when...?",
'DYOR':"Do your own research(common stock market chat slang)",
'E':"Ecstasy",
'E':"Enemy ",
'E1':"Everyone",
'E123':"Easy as one, two, three",
'E2EG':"Ear to ear grin",
'EAK':"Eating at keyboard",
'EBKAC':"Error between keyboard and chair",
'ED':"Erase display",
'EF4T':"Effort",
'EG':"Evil grin",
'EI':"Eat it",
'EIP':"Editing in progress",
'ELI5':"Explain like I'm 5",
'EM':"E-mail",
'EMA':"E-mail address",
'EMBAR': "Embarassing",
'EMFBI':"Excuse me for butting in",
'EMFBI':"Excuse me for jumping in",
'EMSG':"E-mail message",
'ENUF':"Enough",
'EOD':"End of day",
'EOD':"End of discussion",
'EOL':"End of lecture",
'EOL':"End of life",
'EOM':"End of message",
'EOS':"End of show",
'EOT':"End of transmission",
'EQ':"EverQuest ",
'ERP': "Erotic Role-Play"" ",
'ERS2':"Eres tz / are you(Spanish SMS)",
'ES':"Erase screen",
'ESAD':"Eat *S* and die!",
'ETA':"Estimated time(of)arrival",
'ETA':"Edited to add",
'EVA':"Ever",
'EVO':"Evolution",
'EWG':"Evil wicked grin",
'EWI':"Emailing while intoxicated",
'EXTRA':"over the top",
'EYC':"Excitable, yet calm",
'EZ':"Easy",
'EZY':"Easy",
'F':"female",
'F2F':"Face to face",
'F2P':"Free to play ",
'F4F':"Follow for follow",
'FAAK':"Falling asleep at keyboard",
'FAB':"Fabulous",
'Facepalm':"smacking your forehead with your palm",
'FAF':"Funny as *freak*",
'FAM':"friends",
'FAQ':"Frequently asked questions",
'FAY':"*Freak* all you",
'FB':"Facebook",
'FBB': "Facebook buddy",
'FBC':"Facebook chat",
'FBF':"Flashback Friday",
'FBF': "Facebook friend",
'FBF':"Fat boy food",
'FBFR':"FaceBook friend",
'FBM':"Fine by me",
'FBO':"Facebook official",
'FBOW':"For better or worse",
'FC':"Fingers crossed",
'FC':"Full card ",
'FC'INGO':"For crying out loud",
'FCOL':"For crying out loud",
'Feelsbadman':"feel negative.",
'Feelsbatman':"A social meme taking ""feelsbadman"" to the extreme",
'Feelsgoodman':"A social meme that means to feel positive.",
'FEITCTAJ':"*Freak* 'em if they can't take a joke",
'FF':"Follow Friday ",
'FFA':"Free for all ",
'FFS':"For *freak'*sakes",
'FICCL':"Frankly I couldn't care a less",
'FIF':"*Freak* I'm funny",
'FIIK':"*Freaked* if I know",
'FIIOOH':"Forget it, I'm out of here",
'FIL':"Father in law",
'FIMH':"Forever in my heart",
'Finna':"Means ""Going to",
'Finsta':"A second Instagram account where someone can post things that they're too afraid to post on their main account.",
'FISH':"First in, still here",
'FITB':"Fill in the blank",
'FML':"*Freak* My Life",
'FOMC':"Falling off my chair",
'FOMO':"Fear of missing out(definition)",
'FOAD':"*Freak* off and die",
'FOAF':"Friend of a friend",
'FOMCL':"Falling off my chair laughing",
'FRT':"For real though",
'FTBOMH':"From the bottom of my heart",
'FTFY':"Fixed that for you",
'FTL':"For the loss",
'FTW':"For the win",
'FU':"*Freak* you",
'FUBAR':"Fouled up beyond all recognition",
'FUBB':"Fouled up beyond belief",
'FUD':"Face up deal ",
'FUTAB':"Feet up, take a break",
'FW':"Forward",
'FWB':"Friend with benefits",
'FWIW':"For what it's worth",
'FWM':"Fine with me",
'FWP':"First world problems",
'FYE':"Fire, something that is cool",
'FYEO':"For your eyes only",
'FYA':"For your amusement",
'FYI':"For your information",
'G':"Grin",
'G':"Giggle",
'G+':"Google+",
'G/F':"Girlfriend",
'G2CU':"Good to see you",
'G2G':"Got to go",
'G2GICYAL8ER':"Got to go I'll see you later",
'G2R':"Got to run",
'G2TU':"Got to tell you",
'G4C':"Going for coffee",
'G9':"Genius",
'GA':"Go ahead",
'GAC':"Get a clue",
'GAFC':"Get a *freaking* clue",
'GAL':"Get a life",
'GANK':"Meaning a player ambush or unfair player kill ",
'GAS':"Got a second?",
'GAS':"Greetings and salutations",
'GB':"Goodbye",
'GBTW':"Get back to work",
'GBU':"God bless you",
'GD':"Good",
'GDR':"Grinning, ducking, and running",
'GD/R':"Grinning, ducking, and running",
'GFI':"Go for it",
'GF':"Girl friend",
'GFN':"Gone for now",
'GG':"Gotta Go",
'GG':"Good Game ",
'GG':"Brother",
'GGA':"Good game, all ",
'GGE1':"Good game, everyone ",
'GGU2':"Good game, you too",
'GGMSOT':"Gotta get me some of that",
'GGOH':"Gotta Get Outa Here",
'GGP':"Got to go pee",
'GH':"Good hand ",
'GIAR':"Give it a rest",
'GIC':"Gift in crib ",
'GIGO':"Garbage in, garbage out",
'GIRL':"Guy in real life",
'GJ':"Good job",
'GL':"Good luck",
'GL2U':"Good luck to you ",
'GLA':"Good luck all ",
'GL/HF':"Good luck, have fun ",
'GLE':"Good luck everyone ",
'GLE1':"Good luck everyone ",
'GLNG':"Good luck next game ",
'GMBA':"Giggling my butt off",
'GMTA':"Great minds think alike",
'GMV':"Got my vote",
'GN':"Good night",
'GNA':"Good night all",
'GNE1':"Good night everyone",
'GNIGHT':"Good night",
'GNITE':"Good night",
'GNSD':"Good night, sweet dreams",
'GOAT':"Greatest of all Time(s)",
'GOI':"Get over it",
'GOL':"Giggling out loud",
'GOMB':"Get off my back",
'GPOY':"Gratuitous picture of yourself",
'GR8':"Great",
'GRATZ':"Congratulations",
'GRL':"Girl",
'GRWG':"Get right with God",
'GR&D':"Grinning, running and ducking",
'GS':"Good shot",
'GS':"Good split ",
'GT':"Good try",
'GTFO':"Get the *freak* out",
'GTFOH':"Get the *freak* outta here",
'GTG':"Got to go",
'GTM':"Giggling to myself",
'GTRM':"Going to read mail ",
'GTSY':"Great(or good)to see you ",
'GUCCI':"Good",
'GUD':"Good ",
'GWHTLC':"Glad we had this little chat",
'H':"Hug",
'H8':"Hate",
'H8TTU':"Hate to be you",
'HAG1':"Have a good one",
'HAK':"Hug and kiss",
'HALP':"Help(Discord)",
'HAU':"How about you?",
'H&K':"Hugs & kisses",
'H2CUS':"Hope to see you soon",
'HAGN':"Have a good night",
'HAGO':"Have a good one",
'HAND':"Have a nice day",
'HAWT':"Have a wonderful day(out-dated, see next in list)",
'HAWT': "sexy"" or ""attractive",
'HB':"Hurry back",
'HB':"Hug back",
'HBD':"Happy birthday",
'H-BDAY':"Happy Birthday",
'HBU':"How about you?",
'HF':"Have fun",
'HFAC':"Holy flipping animal crackers",
'H-FDAY':"Happy Father's Day",
'HHIS':"Head hanging in shame",
'HIFW':"How I felt when...",
'HL':"Half Life ",
'HLA':"Hola / hello",
'H-MDAY':"Happy Mother's Day",
'HMU':"Hit me up",
'HNL':"(w)Hole 'nother level",
'HOAS':"Hold on a second",
'HP':"Hit points",
'HRU':"How are you?",
'HTH':"Hope this helps",
'HUB':"Head up butt",
'HUYA':"Head up your *butt*",
'HV':"Have ",
'HVH':"Heroic Violet Hold ",
'HW':"Homework",
'HYFR':"Hell yeah, *Freaking* right!",
'I2':"me too",
'IA8':"I already ate",
'IAAA':"I am an accountant",
'IAAD':"I am a doctor",
'IAAL':"I am a lawyer",
'IAC':"In any case",
'IAE':"In any event",
'IANAC':"I am not a crook",
'IANAL':"I am not a lawyer",
'IAO':"I am out(of here)",
'IB':"I'm back",
'IB':"I'm back",
'IC':"I see",
'ICAM':"I couldn't agree more",
'ICBW':"It could be worse",
'ICEDI':"I can't even discuss it",
'ICFILWU':"I could fall in love with you",
'ICYMI':"In case you missed it ",
'IDBI':"I don't believe it",
'IDC':"I don't care",
'IDGAF':" I don't give a *freak*",
'IDK':"I don't know",
'IDTS':"I don't think so",
'IDUNNO':"I don't know",
'IFYP':"I feel your pain",
'IG':"Instagram",
'IG2R':"I got to run",
'IGHT':"I got high tonight",
'IGN':"I got nothing",
'IGP':"I got to(go)pee",
'IHNI':"I have no idea",
'IIRC':"If I remember correctly",
'IIIO':"Intel inside, idiot outside",
'IK':"I know",
'IKR':"I know, right?",
'ILBL8':"I'll be late",
'ILU':"I love you",
'ILUM':"I love you man",
'ILYSM':"I love you so much",
'ILY':"I love you",
'IM':"Instant message",
'IMAO':"In my arrogant opinion",
'IMHO':"In my humble opinion",
'ImL':"Means I love you",
'IMNSHO':"In my not so humble opinion",
'IMO':"In my opinion",
'IMS':"I am sorry",
'IMSB':"I am so bored",
'IMTM':"I am the man",
'IMU':"I miss you",
'INAL':"I'm not a lawyer",
'INC': "incoming"" ",
'Inspoo':"Inspiration",
'INV': "Invite",
'IOMH':"In over my head",
'IOW':"In other words",
'IRL':"In real life",
'IRMC':"I rest my case",
'ISLY':"I still love you",
'ISO':"In search of",
'ITAM':"It's The Accounting, Man",
'ITT':"In This Thread",
'ITYK':"I thought you knew",
'IUSS':"If you say so",
'IWALU':"I will always love you",
'IWAWO':"I want a way out",
'IWIAM':"Idiot wrapped in a moron",
'IWSN':"I want sex now",
'IYKWIM':"If you know what I mean",
'IYO':"In your opinion",
'IYQ': "I like you",
'IYSS':"If you say so",
'j00':"You",
'j00r':"Your",
'JAC':"Just a second",
'JAM':"Just a minute",
'JAS':"Just a second",
'JC(J/C)':"Just checking",
'JDI':"Just do it",
'JELLY': "jealous",
'JFF':"Just for fun",
'JFGI':"Just *freaking* Google it",
'JIC':"Just in case",
'JJ(J/J)':"Just joking",
'JJA':"Just joking around",
'JK(J/K)':"Just kidding",
'JLMK':"Just let me know",
'JMO':"Just my opinion",
'JP':"Just playing ",
'JP':"Jackpot",
'JT(J/T)':"Just teasing",
'JTLYK':"Just to let you know",
'JV':"Joint venture",
'JW':"Just wondering",
'K':"Okay",
'KK':"Knock, knock",
'KK':"Okay, Okay!",
'K8T':"Katie",
'k/b':"Keyboard",
'KB':"Keyboard",
'KB':"Kick butt ",
'KDFU':"Cracking(K)the(D as in Da)*freak* up",
'KEWL':"Cool",
'KEYA':"I will key you later",
'KEYME':"Key me when you get in",
'KFY':"Kiss for you",
'KIA':"Know it all",
'KIR':"Keep it real",
'KISS':"Keep it simple, stupid",
'KIT':"Keep in touch",
'KMA':"Kiss my *a*",
'KMK':"Kiss my keister",
'KMS':"Killing myself",
'KMT':"Kiss my tushie",
'KOC':"Kiss on cheek",
'KOL':"Key opinion leader",
'Koreaboo':"Someone obsessed with Korean culture",
'KOS':"Kid over shoulder",
'KOS':"Kill on sight ",
'KOW':"Knock on wood",
'KOTC':"Kiss on the cheek",
'KOTD':"Kicks of the day(Instagram)",
'KOTL':"Kiss on the lips",
'KNIM':"Know what I mean?",
'KNOW': "knowledge",
'KPC':"Keeping parents clueless",
'KS':"Kill then steal ",
'KSC':"Kind(of)sort(of)chuckle",
'KT':"Katie",
'KUTGW':"Keep up the good work",
'KYS':"Kill yourself",
'L2G':"Like to go?",
'L2G':"Love to go",
'L2K':"Like to come",
'L2P':"Learn to play",
'l33t':"Leet, meaning 'elite'",
'L4L':"Like for like",
'L8R':"Later",
'L8RG8R':"Later, gator",
'LAB':"Life's a *b*",
'LBAY':"Laughing back at you",
'LBS':"Laughing, but serious",
'LBVS':"Laughing, but very serious",
'LD':"Later, dude",
'LD':"Long distance",
'LDO':"Like, duh obviously",
'LEMENO':"Let me know",
'LERK':"Leaving easy reach of keyboard",
'LFD':"Left for day",
'LFG':"Looking for group ",
'LFG':"Looking for guard ",
'LFM':"Looking for more ",
'LGH':"Lets get high",
'LH6':"Lets have sex",
'LHSX':"Lets have sex",
'LHM':"Lord help me",
'LHO':"Laughing head off",
'LI':"LinkedIn",
'LIC':"Like I care",
'LIK':"Meaning liquor",
'LIMT':"Laugh in my tummy",
'LIT':"Meaning really good or something fun and exciting",
'LIT':"Extremely intoxicated",
'LLGB':"Love, later, God bless",
'LLS':"Laughing like *silly*",
'LMAO':"Laughing my *a* off",
'LMBO':"Laughing my butt off",
'LMFAO':"Laughing my freaking *a* off",
'LMIRL':"Lets meet in real life",
'LMK':"Let me know",
'LMMFAO':"Laughing my mother freaking a** off",
'LMNK':"Leave my name out",
'LMS':"Like my status(Facebook)",
'LNT':"lost in translation",
'LOA':"List of acronyms",
'LOL':"Laughing out loud",
'LOL':"Laugh out loud",
'LOL':"Lots of love",
'LOLH':"Laughing out loud hysterically",
'LOLO':"Lots of love",
'LOLWTF':"Laughing out loud ""What the *freak*?",
'LOTI':"Laughing on the inside",
'LOTR':"Lord of The Rings ",
'LQTM':"Laughing quietly to myself",
'LSHMBH':"Laugh so hard my belly hurts",
'LSV':"Language, sex and violence",
'LTD':"Living the dream",
'LTLWDLS':"Let's twist like we did last summer",
'LTNS':"Long time no see",
'LTOD':"Laptop of death",
'LTS':"Laughing to self",
'LULT':"Love you long time",
'LULZ':"Meaning joke, or for laughs",
'LVM':"Left voice mail",
'LWOS':"Laughing without smiling",
'LY':"Love ya ",
'LYLAS':"Love you like a sis",
'LYLC':"Love you like crazy",
'LYSM':"Love you so much",
'M$':"Microsoft",
'M8':"Mate",
'MB':"Mamma's boy",
'MBS':"Mom behind shoulder",
'MC':"Merry Christmas",
'MDIAC':"My Dad is a cop",
'MEGO':"My eyes glaze over",
'MEH':"shrugging shoulders",
'MEH':"just okay",
'MEHH':"sigh",
'MEZ': "mesmerize"" ",
'MFI':"Mad for it",
'MFW':"My face when...",
'MGB':"May God bless",
'MGMT':"Management",
'MHOTY':"My hat off to you",
'MIRL':"Me in real life",
'MIRL':"Meet in real life",
'MISS.( number)': "child and her age",
'MKAY': "Mmm, okay",
'MLM':"Meaning give the middle finger",
'MM':"Sister(Mandarin Chinese txt msg)",
'MMK':"okay?",
'MNC':"Mother nature calls",
'MNSG':"Mensaje",
'MOD':"Moderator",
'MOD':"Modification ",
'MORF':"Male or female?",
'MOMBOY':"Mamma's boy",
'MOO':"My own opinion",
'MOOS':"Member of the opposite sex",
'MOS':"Mother over shoulder",
'MOSS':"Member of same sex",
'MP':"Mana points ",
'MR.': "child and his age"".Mr.3 would be a 3-year old son",
'MRT':"Modified ReTweet",
'MRW':"My reaction when...",
'MSG':"Message",
'MTF':"More to follow",
'MTFBWU':"May the force be with you",
'MU':"Miss U(you)",
'MUAH':"Multiple unsuccessful attempts to humor",
'MUSM':"Miss you so much",
'MWAH': "kiss",
'MYO':"Mind your own business",
'MYOB':"Mind your own business",
'n00b':"Newbie",
'N1':"Nice one",
'N2M':"Nothing too much",
'NADT':"Not a darn thing",
'NALOPKT':"Not a lot of people know that",
'NANA':"Not now, no need",
'NBD':"No big deal",
'NBFAB':"No bad for a beginner ",
'NC':"Nice crib ",
'ND':"Nice double ",
'NE':"Any",
'NE1':"Anyone",
'NERF': "Changed and is now weaker",
'NFM':"None for me / Not for me",
'NFM':"Not for me",
'NGL':"Not gonna(going to)lie",
'NFS':"Need for Speed ",
'NFS':"Not for sale",
'NFW':"No *freaking* way",
'NFW':"Not for work",
'NFWS':"Not for work safe",
'NH':"Nice hand ",
'NIFOC':"Naked in front of computer",
'NIGI':"Now I get it",
'NIMBY':"Not in my back yard",
'NIROK':"Not in reach of keyboard",
'NLT':"No later than",
'NM':"Nothing much",
'NM':"Never mind",
'NM':"Nice meld ",
'NMH':"Not much here",
'NMJC':"Nothing much, just chilling",
'NMU':"Not much, you?",
'NO1':"No one",
'NOOB':"someone who is bad at games",
'NOWL': "knowledge",
'NOYB':"None of your business",
'NP':"No problem",
'NPC':"Non-playing character ",
'NQT':"Newly qualified teacher",
'NR':"Nice roll ",
'NRN':"No response/reply necessary",
'NS':"Nice score ",
'NS':"Nice split ",
'NSA':"No strings attached",
'NSFL':"Not safe for life",
'NSFW':"Not safe for work",
'NSISR':"Not sure if spelled right",
'NT':"Nice try",
'NTHING':"Nothing(SMS)",
'NTS':"Note to self",
'NVM':"Never mind",
'NVR':"Never",
'NW':"No way",
'NWO':"No way out",
'O4U':"Only for you",
'O':"Opponent ",
'O': "hugs",
'O':"Over",
'OA':"Online auctions",
'OATUS':"On a totally unrelated subject",
'OB':"Oh baby",
'OB':"Oh brother",
'OBV':"Obviously",
'OFC':"Of course",
'OG':"Original gangster",
'OGIM':"Oh God, it's Monday",
'OH':"Overheard",
'OHHEMMGEE': "Oh My God",
'OI':"Operator indisposed",
'OIB':"Oh, I'm back",
'OIC':"Oh, I see",
'OJ':"Only joking",
'OL':"Old lady",
'OLL':"Online love",
'OM':"Old man",
'OM':"Oh, my",
'OMAA':"Oh, my aching *A*(butt)",
'OMDB':"Over my dead body",
'OMFG':"Oh my *freaking* God",
'omfglmaobbqrofl':"oh my *freaking* god, laugh my *a* off, owned, roll on floor spinning around I'm so sad",
'copteriss':"nan",
'OMG':"Oh my God",
'OMG':"Oh my gosh",
'OMGYG2BK':"Oh my God, you got to be kidding",
'OMGYS':"Oh my gosh you suck ",
'OMS':"On my soul(meaning ""promise"")",
'OMW':"On my way",
'ONL':"Online",
'OO':"Over and out",
'OOC':"Out of character",
'OOH':"Out of here",
'OOMF':"One of my followers",
'OOTD':"One of these days",
'OOTD':"Outfit of the day",
'OOTO':"Out of the office",
'OP':"On phone",
'ORLY':"Oh really?",
'OS':"Operating system",
'OT':"Off topic",
'OTB':"Off to bed",
'OTFL':"On the floor laughing",
'OTL':"Out to lunch",
'OTOH':"On the other hand",
'OTP':"On the phone",
'OTP':"One true pairing; two people you would love to see as a couple",
'OTT':"Over the top",
'OTTOMH':"Off the top of my head",
'OTW':"Off to work",
'OVA':"Over",
'OYO':"On your own",
'P':"Partner ",
'P2P':"Parent to parent",
'P2P':"Peer to peer",
'P2P':"Pay to play ",
'P911':"Parents coming into room alert",
'PAP':"Post a picture",
'PAT': "patrol"" ",
'PAW':"Parents are watching",
'PBOOK':"Phonebook(e-mail)",
'PC':"Player character ",
'PCM':"Please call me",
'PDA':"Personal display(of)affection",
'PDH':"Pretty darn happy",
'PDS':"Please don't shoot",
'PDQ':"Pretty darn quick",
'PEEPS':"People",
'PFT':"Pretty *freaking* tight",
'PIC':"Picture",
'PIP':"Peeing in pants",
'PIR':"Parents in room",
'PISS':"Put in some sugar",
'PITA':"Pain in the *butt*",
'PKMN':"Pokemon ",
'PL8':"Plate",
'PLD':"Played",
'PLMK':"Please let me know",
'PLS':"Please",
'PLU':"People like us",
'PLZ':"Please",
'PLZTLME':"Please tell me",
'PM':"Private Message",
'PMFI':"Pardon me for interrupting",
'PMFJI':"Pardon me for jumping in",
'PMSL':"Pee myself laughing",
'POAHF':"Put on a happy face",
'POIDH':"Picture, or it didn't happen",
'POS':"Parent over shoulder",
'POT': "potion"" ",
'POTD':"Photo of the day",
'POV':"Point of view",
'POV':"Privately owned vehicle",
'PPL':"People",
'PPU':"Pending pick-up",
'PRESH':"Precious",
'PROBS':"Probably",
'PROLLY':"Probably",
'PROGGY':"Meaning computer program",
'PRON':"Meaning pornography",
'PRT':"Party",
'PRT':"Please Retweet ",
'PRW':"People/parents are watching",
'PSA':"Public Service Announcement",
'PSOS':"Parent standing over shoulder",
'PSP':"Playstation Portable",
'PST':"Please send tell ",
'PTFO':"Pass the *freak* out",
'PTIYPASI':"Put that in your pipe and smoke it",
'PTL':"Praise the Lord",
'PTMM':"Please tell me more",
'PTO':"Paid time off",
'PTO':"Personal time off",
'PTO':"Parent Teacher Organization",
'PUG':"Pick up group ",
'PVE':"Player vs enemy, Player versus environment ",
'PVP':"Player versus player ",
'PWN': "own",
'PXT':"Please explain that",
'PU':"That stinks!",
'PUKS':"Pick up kids(SMS)",
'PYT':"Pretty young thing",
'PZ':"Peace ",
'PZA':"Pizza",
'Q':"Queue",
'Q4U':"(I have a)question for you",
'QC':"Quality control",
'QFE':"Question for everyone",
'QFI':"Quoted for idiocy",
'QFI':"Quoted for irony",
'QFT':"Quoted for truth",
'QIK':"Quick",
'QL':"Quit laughing",
'QOTD':"Quote of the day",
'QQ(qq )(Q_Q)': "crying eyes",
'QQ':"Quick question",
'QSL':"Reply ",
'QSO':"Conversation",
'QT':"Cutie",
'QTPI':"Cutie pie",
'R': "are",
'R8':"Rate(SMS)",
'RBAY':"Right back at you",
'RFN':"Right *freaking* now",
'RGR':"I agree",
'RHIP':"Rank has its privileges",
'RIP':"Rest in peace",
'RL':"Real life",
'RLY':"Really",
'RME':"Rolling my eyes",
'RMLB':"Read my lips baby",
'RMMM':"Read my mail man",
'ROFL':"Rolling on floor laughing",
'ROFLCOPTER':"Rolling on floor laughing and spinning around",
'ROFLMAO':"Rolling on the floor, laughing my *butt* off",
'ROTFL':"Rolling on the floor laughing",
'ROTFLUTS':"Rolling on the floor laughing unable to speak",
'RS':"Runescape ",
'RSN':"Real soon now",
'RT':"Roger that",
'RT':"Retweet ",
'RTBS':"Reason to be single",
'RTFM':"Read the *freaking* manual",
'RTFQ':"Read the *freaking* question",
'RTHX': "Thanks for the RT(Retweet)"" ",
'RTMS':"Read the manual, stupid",
'RTNTN':"Retention",
'RTRCTV':"Retroactive",
'RTRMT':"Retirement",
'RTSM':"Read the stupid manual",
'RTWFQ':"Read the whole *freaking* question",
'RU':"Are you?",
'RUMOF':"Are you male or female?",
'RUT':"Are u(you)there?",
'RUOK':"Are you okay?",
'RX':"Regards",
'RW':"Real world",
'RX':"Meaning drugs or prescriptions",
'RYB':"Read your Bible ",
'RYO':"Roll your own",
'RYS':"Read your screen ",
'RYS':"Are you single?",
'S2R':"Send to receive(meaning send me your picture to get mine)",
'S2S':"Sorry to say",
'S4L':"Spam for life",
'SAL':"Such a laugh",
'SAT':"Sorry about that",
'SAVAGE':"Slang for a shockingly careless expression or response to an event or message",
'SB':"Snap back",
'SB':"Should be",
'SB':"Smiling back",
'SBIA':"Meaning Standing back in amazement",
'SBT':"Sorry 'bout that",
'SC':"Stay cool",
'SD':"Sweet dreams",
'SDMB':"Sweet dreams, my baby",
'SENPAI':"Meaning someone older than you, someone you look up to",
'SEO':"Search engine optimization",
'SETE':"Smiling Ear-to-Ear",
'SELFIE':"A photo that is taken of oneself for social media sharing(definition)",
'SFAIK':"So far as I know",
'SH':"Same here",
'SH^':"Shut up",
'SHID':"Slapping head in disgust",
'SHIP':"Slang for ""wishing two people were in a relationship",
'SICNR':"Sorry, I could not resist",
'SIG2R':"Sorry, I got to run",
'SIHTH':"Stupidity is hard to take",
'SIMYC':"Sorry I missed your call",
'SIR':"Strike it rich",
'SIS':"Snickering in silence",
'SIS':"Meaning sister",
'SIT':"Stay in touch",
'SITD':"Still in the dark",
'SJW':"Social justice warrior",
'SK8':"Skate",
'SK8NG':"Skating",
'SK8R':"Skater",
'SK8RBOI':"Skater Boy",
'SLAP':"Sounds like a plan",
'Slay':"To succeed at something",
'SM':"Social media",
'SMAZED':"Smoky haze",
'SMEXI':"Combination of sexy and Mexican, used to describe attractive people",
'SMH':"Shaking my head",
'SMHID':"Scratching my head in disbelief",
'SNAFU':"Situation normal all fouled up",
'SNERT':"Snot nosed egotistical rude teenager",
'SNR':"Streaks and Recents(Snapchat)",
'SO':"Significant other",
'SOAB':"Son of a *B*",
'S'OK':"Meaning It'(s)okay ",
'SOL':"Sooner or later",
'SOMY':"Sick of me yet?",
'SorG':"Straight or Gay?",
'SOS':"Meaning help",
'SOS':"Son of Sam",
'SOT':"Short of time",
'SOTMG':"Short of time, must go",
'SOWM':"Someone with me",
'SPK':"Speak(SMS)",
'SRSLY':"Seriously",
'SPST':"Same place, same time",
'SPTO':"Spoke to",
'SQ':"Square",
'SRY':"Sorry",
'SS':"So sorry",
'SSDD':"Same stuff, different day",
'SSIF':"So stupid it's funny",
'SSINF':"So stupid it's not funny",
'ST&D':"Stop texting and drive",
'Stan':"Meaning a die-hard fan of someone",
'STFU':"Shut the *freak* up",
'STR8':"Straight",
'STW':"Search the Web",
'SU':"Shut up",
'SUITM':"See you in the morning",
'SUL':"See you later",
'SUP':"What's up?",
'SUTH':"So use to haters",
'SUX':"Meanings sucks or ""it sucks",
'SUYF':"Shut up you fool",
'SWAG':"Meaning free stuff and giveaways from tech tradeshows",
'SWAG':"Scientific wild *a* guess",
'SWAK':"Sent with a kiss",
'SWALK':"Sealed with a loving kiss",
'SWAT':"Scientific wild *butt* guess",
'SWL':"Screaming with laughter",
'SWMBO':"She who must be obeyed.Meaning wife or partner",
'SYL':"See you later ",
'SYS':"See you soon ",
'SYY':"Shut your yapper",
'T+':"Think positive",
'T4BU':"Thanks for being you",
'T:)T':"Think happy thoughts",
'TA':"Thanks a lot",
'TAFN':"That's all for now",
'TAM':"Tomorro a.m.",
'TANK':"really strong",
'TANKED': "owned",
'TANKING': "owning",
'TARFU':"Things Are Really *fouled* Up.",
'TAU':"Thinking about you",
'TAUMUALU':"Thinking about you miss you always love you",
'TBAG':"Process of disgracing a corpse, taunting a fragged/killed player ",
'TBBH':"To be brutally honest",
'TBC':"To be continued",
'TBD':"To be determined",
'TBH':"To be honest",
'TBL':"Text back later",
'TBT':"Throwback Thursday ",
'TC':"Take care",
'TCB':"Take care of business",
'TCOY':"Take care of yourself",
'TD':"Tower defense ",
'TD2M':"Talk dirty to me",
'TDTM':"Talk dirty to me",
'TEA':"Meaning gossip",
'TFF':"Too *freaking* funny",
'TFS':"Thanks for sharing",
'TFTF':"Thanks for the follow ",
'TFTI':"Thanks for the invitation",
'TFTT':"Thanks for this tweet ",
'TG':"Thank goodness",
'TGIF':"Thank God it's Friday",
'THNQ':"Thank-you(SMS)",
'THNX':"Thanks",
'THOT':"That wh*re over there",
'THT':"Think happy thoughts",
'THX':"Thanks",
'TIA':"Thanks in advance",
'TIAD':"Tomorrow is another day",
'TIC':"Tongue-in-cheek",
'TIL':"Today I learned",
'TILIS':"Tell it like it is",
'TIR':"Teacher in room",
'TLK2UL8R':"Talk to you later",
'TL':"Too long",
'TL;DR':"Too long; didn't read",
'TM':"Trust me",
'TMA':"Take my advice",
'TMB':"Text me back",
'TMB':"Tweet me back ",
'TMI':"Too much information",
'TMOT':"Trust me on this",
'TMTH':"Too much to handle",
'TMYL':"Tell me your location",
'TMWFI':"Take my word for it",
'TNSTAAFL':"There's no such thing as a free lunch",
'TNT':"Til next time",
'TOJ':"Tears of joy",
'TOS':"Terms of service",
'TOTES':"Totally",
'TOU':"Thinking of you",
'TOY':"Thinking of you",
'TPM':"Tomorrow p.m.",
'TPTB':"The powers that be",
'TQ':"Te quiero / I love you(Spanish SMS)",
'TSH':"Tripping so hard",
'TSNF':"That's so not fair",
'TSTB':"The sooner, the better",
'TT':"Trending topic ",
'TTFN':"Ta ta for now",
'TTLY':"Totally",
'TTTT':"These things take time",
'TTUL':"Talk to you later",
'TU':"Thank you",
'TUI':"Turning you in",
'TURNT':"Meaning excitement, turned up",
'TWSS':"That's what she said",
'TTG':"Time to go",
'TTYAFN':"Talk to you awhile from now",
'TTYL':"Talk to you later",
'TTYS':"Talk to you soon",
'TY':"Thank you",
'TYFC':"Thank you for charity ",
'TYFYC':"Thank you for your comment",
'TYS':"Told you so",
'TYT':"Take your time",
'TYSO':"Thank you so much",
'TYAFY':"Thank you and *freak* you",
'TYVM':"Thank you very much",
'TX':"Thanks",
'^URS':"Up yours",
'UCMU':"You crack me up",
'UDI':"Unidentified drinking injury(meaning bruise, scratch, ache and so on)",
'UDM':"You the man",
'UDS':"Ugly domestic scene",
'UFB':"Un *freaking* believable",
'UFN':"Until further notice",
'UFWM':"You *freaking* with me?",
'UGTBK':"You've got to be kidding",
'UHGTBSM':"You have got to be s
'UKTR':"You know that's right",
'UL':"Upload",
'U-L': "You will",
'UNA':"Use no acronyms",
'UN4TUN8':"Unfortunate",
'UNBLEFBLE':"Unbelievable",
'UNCRTN':"Uncertain",
'UNPC':"Un-(not)politically correct",
'UOK':"Are You ok?",
'UR':"You are / You're",
'UR2YS4ME':"You are too wise for me",
'URA*':"You are a star",
'URH':"You are hot(U R Hot)",
'URSKTM':"You are so kind to me",
'URTM':"You are the man",
'URW':"You are welcome",
'USBCA':"Until something better comes along",
'USU':"Usually",
'UT':"Unreal Tournament ",
'UT2L':"You take too long",
'UTM':"You tell me",
'UV':"Unpleasant visual",
'UW':"You're welcome",
'UX':"User experience",
'V':"Very",
'VAT':"Value added tax",
'VBL':"Visible bra line",
'VBS':"Very big smile",
'VC':"Voice chat",
'VEG':"Very evil grin",
'VFF':"Very freaking funny",
'VFM':"Value for money",
'VGC':"Very good condition",
'VGG':"Very good game ",
'VGH':"Very good hand ",
'VIP':"Very important person",
'VM':"Voice mail",
'VN':"Very nice",
'VNH':"Very nice hand ",
'VoIP':"Voice over Internet Protocol(definition)",
'VRY':"Very",
'VSC':"Very soft chuckle",
'VSF':"Very sad face",
'VWD':"Very well done ",
'VWP':"Very well played ",
'W@':"What?",
'W/':"With",
'W/B':"Welcome back",
'W3':"WWW(Web address)",
'W8':"Wait",
'WAH':"Working at home",
'WAJ':"What a jerk",
'WAM':"Wait a minute",
'WAN2':"Want to?(SMS)",
'WAN2TLK':"Want to talk",
'WAREZ':"Meaning pirated(illegally gained)software",
'WAS':"Wait a second",
'WAS':"Wild *a* guess",
'WAT':"What",
'WAWA':"Where are we at?",
'WAYF':"Where are you from?",
'W/B':"Write back",
'WB':"Welcome back",
'WBS':"Write back soon",
'WBU':"What about you?",
'WC':"Welcome",
'WC':"Who cares",
'WCA':"Who cares anyway",
'W/E':"Whatever",
'W/END':"Weekend",
'WE':"Whatever",
'WEBO':"Webopedia",
'WEEBO':"Describes a person obsessed with of Japanese culture",
'WEP':"Weapon ",
'WH5':"Who, what, when, where, why",
'WHATEVES':"Whatever",
'WIBNI':"Wouldn't it be nice if",
'WDALYIC':"Who died and left you in charge",
'WDYK':"What do you know?",
'WDYT':"What do you think?",
'WGACA':"What do you think?",
'WIIFM':"What's in it for me?",
'WISP':"Winning is so pleasurable",
'WITP':"What is the point?",
'WITW':"What in the world",
'WIU':"Wrap it up",
'WK':"Week",
'WKD':"Weekend",
'WRT':"With regard to",
'WL':"Whatta loser",
'W/O':"Without",
'WOA':"Work of Art",
'WOKE':"Slang for people who are aware of current social issues, and politics",
'WOMBAT':"Waste of money, brains and time",
'WOW':"World of Warcraft ",
'WRK':"Work",
'WRU':"Where are you?",
'WRU@':"Where are you at?",
'WRUD':"What are you doing?",
'WTB':"Want to buy ",
'WTF':"What the *freak* ?",
'WTFE':"What the *freak* ever",
'WTFO':"What the *freak* ?, over.",
'WTG':"Way to go",
'WTGP':"Want to go private(talk out of public chat area)",
'WTH':"What the heck?",
'WTM':"Who's the man?",
'WTS':"Want to sell? ",
'WTT':"Want to trade? ",
'WU':"What's up?",
'WUCIWUG':"What you see is what you get",
'WUF':"Where are you from?",
'WUP':"What's up?",
'WUT': "what",
'WUW':"What u(you)want?",
'WUU2':"What are you up to?",
'WUZ': "was",
'WWJD':"What would Jesus do?",
'WWJD':"What Would Judd Do?(Artist Donald Judd, who was known to do things in his own compulsive way.) ",
'WWNC':"Will wonders never cease",
'WWYC':"Write when you can",
'WYCM':"Will you call me?",
'WYD':"What(are)you doing?",
'WYGAM':"When you get a minute",
'WYHAM':"When you have a minute",
'WYLEI':"When you least expect it",
'WYSIWYG':"What you see is what you get ",
'WYWH':"Wish you were here",
'X-1-10': "Exciting",
'X':"Kiss",
'X!': "a typical woman",
'XD': "really hard laugh""(where D is a smiley mouth)",
'XD':"Meaning a ""devilish smile",
'XME':"Excuse Me",
'XOXOXO':"Hugs & Kisses",
'XLNT':"Excellent ",
'XLR8': "faster"" or ""going faster",
'XPOST':"Meaning Cross-post.A link posted to a subreddit that was already shared on a different subreddit(Reddit)",
'XYL':"Ex-young lady, meaning wife.( amateur radio)",
'XYZ':"Examine your zipper",
'Y?':"Why?",
'Y':"Meaning Yawn",
'Y2K':"You're too kind",
'YA':"Your",
'YAA':"Yet another acronym",
'YABA':"Yet another bloody acronym",
'YARLY':"Ya, really?",
'YAS': "praise",
'YBIC':"Your brother in Christ",
'YBS':"You'll be sorry",
'YCDBWYCID':"You can't do business when your computer is down",
'YCHT':"You can have them",
'YCLIU':"You can look it up",
'YCMU':"You crack me up",
'YCT':"Meaning Your comment to?",
'YD':"Yesterday",
'YEET':"Meaning excitement, approval or display of energy(i.e.throwing something)",
'YF':"Wife",
'YG':"Young gentleman",
'YGG':"You go girl",
'YGTBKM':"You've got to be kidding me",
'YGTR':"You got that right",
'YHBT':"You have been trolled",
'YHBW':"You have been warned",
'YHL':"You have lost",
'YIU':"Yes, I understand",
'YKW':"You know what",
'YKWYCD':"You know what you can do",
'YL':"Young lady",
'YMMV':"Your mileage may vary",
'YNK':"You never know",
'YOLO':"You only live once",
'YR':"Your",
'YR':"Yeah right",
'YRYOCC':"You're running your own cuckoo clock",
'YSIC':"Your sister in Christ",
'YSYD':"Yeah sure you do",
'YT':"YouTube",
'YT':"You there?",
'YTB':"You're the best",
'YTB':"Youth talk back",
'YTTL':"You take too long",
'YTG':"You're the greatest",
'YW':"You're welcome",
'YWHNB':"Yes, we have no bananas",
'YWHOL':"Yelling ""woohoo"" out loud",
'YWSYLS':"You win some, you lose some",
'YYSSW':"Yeah, yeah, sure, sure, whatever",
'Z':"Zero",
'Z':"Z's are calling(meaning going to bed/sleep)",
'Z': "Said",
'Z%':"Zoo",
'ZH':"Sleeping Hour",
'ZOMG':"Used in World of Warcraft to mean OMG(Oh My God)",
'ZOT':"Zero tolerance",
'ZUP': "What's up?",
'ZZZZ':"Sleeping(or bored)"
}<categorify>
categorical_features_to_impute_none += ['MasVnrType']
def convert_Abbreviation(text):
for abb in Abbreviation.items() :
if(not abb[0].isdigit())and(" "+abb[0]+" " in text)and(len(abb[0])>3):
text = text.replace(abb[0],abb[1])
 return text 
<define_variables>
hp[hp['BsmtQual'].isna() ][['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','BsmtFinSF1',
'BsmtFinSF2','BsmtUnfSF','TotalBsmtSF']]
bracket_pattern = re.compile('\[|\]')
 time_pattern = re.compile('\d+(( \\|\/)\d+)+')
 number_pattern = re.compile('\d+(,|\.|\d+)*')
 including_number_pattern = re.compile('\s\d+(\W*\d)*\s')
 hashtag_pattern = re.compile('
alpha_pattern = re.compile('@')
 remove_except_chracter_pattern = re.compile('[ ](?=[ ])|[^A-Za-z\s]+')
<categorify>
hp[bsmt_cols].isna().sum()
def remove_reg(text, reg):
if reg == time_pattern:
return reg.sub(r' TIME ', text)
 elif reg == number_pattern:
return reg.sub(r'00', text)
 else:
return reg.sub(r' ', text )<drop_column>
categorical_features_to_impute_none += bsmt_cols
remove_reg("sd1:2f",time_pattern )<categorify>
hp[hp['Electrical'].isna() ]
repeated_pattern = re.compile(r'(\!|\? )(( \!|\?){1,})')
 unnecessary_pattern = re.compile(r"'|"")
 def remove_punctuation(text):

 text = unnecessary_pattern.sub(r' ', text)
 return repeated_pattern.sub(r'\1', text )<drop_column>
garage_cols = ['GarageType','GarageFinish','GarageQual','GarageCond']
hp[hp['GarageType'].isna() ][garage_cols]
remove_punctuation("1!! !!! ???? ?<categorify>
hp[garage_cols].isna().sum()
redundant_white_spaces_pattern = re.compile(r'\s+')
 redundant_url = re.compile(r'url(\s*url)+')
 redundant_id = re.compile(r'id(\s*id)+')
 def remove_redundant_white_spaces(text):
text = redundant_url.sub(r'\1', text)
 text = redundant_id.sub(r'\1', text)
 text = text.strip()
text = redundant_white_spaces_pattern.sub(r' ', text)
 return text<categorify>
categorical_features_to_impute_none += garage_cols
reg_list = [bracket_pattern, number_pattern, time_pattern, including_number_pattern, hashtag_pattern, alpha_pattern, remove_except_chracter_pattern]
def preprocessor(dataset, function_list, columns):
for column in columns:
cleaned_colname = column+"_cleaned"
dataset[cleaned_colname] = dataset[column]
for i, function in enumerate(function_list):
if function in reg_list:
dataset[cleaned_colname] = dataset[cleaned_colname].apply(lambda text: remove_reg(text, function)if text is not np.nan else "")
 print(i,")",function, "is done.")
 else:
dataset[cleaned_colname] = dataset[cleaned_colname].apply(lambda text: function(text)if text is not np.nan else "")
 print(i,")",function.__name__, "is done.")
 dataset[cleaned_colname].replace('', np.nan, inplace=True)
 print("For", column, "column, preprocessing is done.")
 return None<define_variables>
categorical_features_to_impute_mode = list(set(categorical_oh_features + categorical_l_features ).difference(set(categorical_features_to_impute_none)))
 categorical_features_to_impute_mode
function_list = [
remove_words_containing_alpha_by_threshold,
clean_1_2_1,clean_1_2_2,clean_1_2_3,clean_1_2_4, clean_1_2_6,clean_1_2_7,clean_1_2_8,
remove_url,
convert_emoticons,
mylower,
bracket_pattern,
time_pattern,
including_number_pattern,
hashtag_pattern,
alpha_pattern,
remove_punctuation,
remove_redundant_white_spaces
]
preprocess_target_cols = ['text']<feature_engineering>
categorical_none_imputer = impute.SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='ImputedNone')
 hp[categorical_features_to_impute_none] = categorical_none_imputer.fit_transform(hp[categorical_features_to_impute_none] )
def compare_2col(df, target_colname, base_colname):
result_colname = target_colname+"_from_"+base_colname
df[result_colname] = df.apply(lambda row :
row[base_colname] != row[target_colname], axis=1)
 total = df.shape[0]
changed = len(df[df[result_colname]==True])
 print(f'{base_colname}에서 {target_colname}가 되면서
전체 {total}개 중 {changed}개({(changed/total)*100 :0.1f}%)가 변함')
 return None
def check_cleaning(df, func, target_colname, base_colname,print_num=10):
result_colname = target_colname+"_from_"+base_colname
if func != None:
if func in reg_list:
df[target_colname] = df[base_colname].apply(lambda text: remove_reg(text, func))
print(func, "is done.")
 else:
df[target_colname] = df[base_colname].apply(lambda text: func(text))
print(func.__name__, "is done.")
 compare_2col(df, target_colname, base_colname)
 display(df.loc[df[result_colname]==True, ['target',base_colname,target_colname]].head(print_num))
df.drop(columns=[result_colname], axis=1, inplace=True)
 return None 
<categorify>
categorical_mode_imputer = impute.SimpleImputer(missing_values=np.nan, strategy='most_frequent')
 hp[categorical_features_to_impute_mode] = categorical_mode_imputer.fit_transform(hp[categorical_features_to_impute_mode] )
df_temp = dataset_origin_df.copy()
func_num = 14
preprocessor(df_temp, function_list[:func_num], preprocess_target_cols)
 df_temp['text'] = df_temp['text_cleaned']<compute_test_metric>
hp.isna().sum().sum()
check_cleaning(df_temp, function_list[func_num],'text_cleaned', 'text',100)
<filter>
hp[apply_log] = np.log(hp[apply_log] )
df_temp[df_temp['text'].str.contains('camilacabello97', na=False, case=False)]<categorify>
def create_features(df):
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalFeet'] = df['BsmtFinSF1'] + df['BsmtFinSF2'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalBath'] = df['FullBath'] +(0.5 * df['HalfBath'])+ df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath'])
 df['TotalPorch'] = df['OpenPorchSF'] + df['3SsnPorch'] + df['EnclosedPorch'] + df['ScreenPorch'] + df['WoodDeckSF']
df['HasPool'] = df['PoolArea'].apply(bigger_than_zero)
 df['Has2ndFlr'] = df['2ndFlrSF'].apply(bigger_than_zero)
 df['HasGarage'] = df['GarageArea'].apply(bigger_than_zero)
 df['HasBsmt'] = df['TotalBsmtSF'].apply(bigger_than_zero)
 df['HasFireplace'] = df['Fireplaces'].apply(bigger_than_zero)
preprocessor(train_df, function_list, preprocess_target_cols)
 preprocessor(test_df, function_list, preprocess_target_cols )<drop_column>
create_features(hp )
def remove_cleaned_col(df):
for col in preprocess_target_cols:
cleaned_colname = col+"_cleaned"
df[col] = df[cleaned_colname]
df.drop(columns=[cleaned_colname], axis=1, inplace=True)
 remove_cleaned_col(train_df)
 remove_cleaned_col(test_df )<load_from_csv>
one_hot_encoder = p.OneHotEncoder(sparse=False, handle_unknown='ignore')
 hp_oh = pd.DataFrame(one_hot_encoder.fit_transform(hp[categorical_oh_features]), columns=one_hot_encoder.get_feature_names(categorical_oh_features))
hp_oh
mislabeled_corrected_df = pd.read_csv('/kaggle/input/nlp-wdt-hapjeong/mislabeled_corrected_V19.csv')
<feature_engineering>
hp_oh.index = hp.index
hp.drop(categorical_oh_features, axis=1, inplace=True)
 hp = pd.concat([hp, hp_oh], axis=1)
 hp
i=0
for index, row in mislabeled_corrected_df[mislabeled_corrected_df['target'].isin([0.0,1.0])].iterrows() :
i=i+1
train_df.loc[train_df['id'] == row['id'], 'target'] = int(row['target'])
 print(i )<filter>
ordinal_encoder = p.OrdinalEncoder()
hp[categorical_l_features] = pd.DataFrame(ordinal_encoder.fit_transform(hp[categorical_l_features]), columns=hp[categorical_l_features].columns)
 hp[categorical_l_features]
i=0
for index, row in mislabeled_corrected_df[~mislabeled_corrected_df['target'].isin([0.0,1.0])].iterrows() :
i=i+1
train_df = train_df[train_df['id'] != row['id']]
print(i )<save_to_csv>
hp.isna().sum().sum()
train_df.to_csv('train_cleaned.csv', index=False)
 test_df.to_csv('test_cleaned.csv', index=False)
<import_modules>
isolation_forest = e.IsolationForest()
isolation_forest.fit(hp)
 outliers_res = isolation_forest.predict(hp )
import numpy as np
import pandas as pd
import time
from datetime import datetime<load_from_csv>
outliers = hp.iloc[outliers_res == -1].index
train = pd.read_csv('train_cleaned.csv')
 test = pd.read_csv('test_cleaned.csv' )<import_modules>
t = hp.drop(outliers, axis=0)
 t.isna().sum().sum()
hp = t
from google.cloud import storage, automl_v1beta1 as automl
from google.api_core.gapic_v1.client_info import ClientInfo
from automlwrapper import AutoMLWrapper<choose_model_class>
hp.isna().sum().sum()
PROJECT_ID = 'kaggle-nlp-wdt'
BUCKET_NAME = 'kaggle-nlp-wdt-lcm'
region = 'us-central1'
storage_client = storage.Client(project=PROJECT_ID)
 client = automl.AutoMlClient(client_info=ClientInfo())
 print(f'Starting AutoML notebook at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )<define_variables>
y = hp.SalePrice
VERSION = 'V19'
BUCKET_PATH = 'preprocessing/'+VERSION+'/'
FILE_NAME = 'train_cleaned'+'_'+VERSION
training_gcs_path = BUCKET_PATH+FILE_NAME+'.csv'
dataset_display_name = FILE_NAME
model_display_name = 'model_'+FILE_NAME<save_to_csv>
X = hp.drop(columns='SalePrice' )
train.loc[:,['text','target']].to_csv('train.csv', index=False, header=False )<prepare_output>
X_train, X_hold_out, X_val = np.split(X, [int (.7*len(X)) , int (.9*len(X)) ])
 y_train, y_hold_out, y_val = np.split(y, [int (.7*len(y)) , int (.9*len(y)) ] )
bucket = storage.Bucket(storage_client, name=BUCKET_NAME)
 if not bucket.exists() :
bucket.create(location=BUCKET_REGION )<load_pretrained>
hp.isna().sum().sum()
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}'.format(
 source_file_name,
'gs://' + bucket_name + '/' + destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )<load_from_csv>
upload_blob(BUCKET_NAME, 'train.csv', training_gcs_path)
<choose_model_class>
scaler = p.StandardScaler()
X_train = pd.DataFrame(scaler.fit_transform(X_train),columns=X_train.columns)
 X_hold_out = pd.DataFrame(scaler.transform(X_hold_out),columns=X_hold_out.columns)
 X_val = pd.DataFrame(scaler.transform(X_val),columns=X_val.columns)
 X = pd.DataFrame(scaler.transform(X),columns=X.columns)
 X
amw = AutoMLWrapper(client=client,
project_id=PROJECT_ID,
bucket_name=BUCKET_NAME,
region='us-central1',
dataset_display_name=dataset_display_name,
model_display_name=model_display_name)
<prepare_x_and_y>
print(f'Getting dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_dataset_by_display_name(dataset_display_name):
print('dataset not found')
 amw.create_dataset()
amw.import_gcs_data(training_gcs_path)
 amw.dataset
print(f'Dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
<train_model>
print(f'Getting model trained at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_model_by_display_name(model_display_name):
print(f'Training model at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.train_model()
print(f'Model trained.Ensuring model is deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.deploy_model()
amw.model
print(f'Model trained and deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
<find_best_params>
regressor_svr = svm.SVR(kernel='rbf', C=2, epsilon=0.01, gamma=0.0005)
 regressor_en = lm.ElasticNet(alpha=0.1, l1_ratio=0.025)
 regressor_gbr = e.GradientBoostingRegressor(learning_rate=0.05, max_depth=3, n_estimators=1200)
 regressor_lgbm = lgb.LGBMRegressor(lambda_l1=0.01, lambda_l2=0.1, learning_rate=0.05,
min_data_in_leaf=25, n_estimators=2000, num_leaves=4)
 regressor_xgb = xgb.XGBRegressor(learning_rate=0.01, max_depth=4, n_estimators=4000)
 ensemble_of_regressors = [regressor_svr, regressor_en, regressor_gbr, regressor_lgbm, regressor_xgb]
amw.model_full_path<predict_on_test>
for clf in ensemble_of_regressors:
clf.fit(X_train, y_train )
print(f'Begin getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 prediction_client = automl.PredictionServiceClient()
amw.set_prediction_client(prediction_client)
 predictions_df = amw.get_predictions(test,
input_col_name='text',
limit=None,
threshold=0.5,
verbose=False)
 print(f'Finished getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )<concatenate>
def get_predictions(x, ensemble_of_regressors, y=None):
pred_result = pd.DataFrame()
i = 1
for clf in ensemble_of_regressors:
y_pred = clf.predict(x)
 if y is not None:
print(clf.__class__.__name__, m.r2_score(y, y_pred))
pred_result.insert(i - 1, 'y_pred_' + str(i), y_pred)
 i += 1
return pred_result
submission_df = pd.concat([test['id'], predictions_df['class']], axis=1)
 submission_df.head()<rename_columns>
pred_result = get_predictions(X_hold_out, ensemble_of_regressors, y_hold_out )
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()<save_to_csv>
X_stack_train = pred_result
y_stack_train = y_hold_out
submission_df.to_csv("submission.csv", index=False, header=True )<set_options>
blender_of_ensamble = lm.LinearRegression()
blender_of_ensamble.fit(X_stack_train, y_stack_train )
nltk.download('stopwords', quiet=True)
 stopwords = stopwords.words('english')
 sns.set(style="white", font_scale=1.2)
 plt.rcParams["figure.figsize"] = [10,8]
pd.set_option.display_max_columns = 0
pd.set_option.display_max_rows = 0<load_from_csv>
pred_result_test = get_predictions(X_val, ensemble_of_regressors, y_val )
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )<feature_engineering>
X_stack_test = pred_result_test
y_stack_pred = blender_of_ensamble.predict(X_stack_test)
 X_stack_test.shape, y_stack_pred.shape
null_counts = pd.DataFrame({"Num_Null": train.isnull().sum() })
 null_counts["Pct_Null"] = null_counts["Num_Null"] / train.count() * 100
null_counts<count_values>
m.r2_score(y_stack_pred, y_val )
len(train["keyword"].value_counts() )<feature_engineering>
estimators2 = [ 
('svr', svm.SVR(kernel='rbf', C=2, epsilon=0.01, gamma=0.0005)) , 
('en', lm.ElasticNet(alpha=0.1, l1_ratio=0.03)) , 
('gbr', e.GradientBoostingRegressor(learning_rate=0.05, max_depth=3, n_estimators=1200)) , 
('lgbm', lgb.LGBMRegressor(lambda_l1=0.01, lambda_l2=0.1, learning_rate=0.05,
min_data_in_leaf=25, n_estimators=2000, num_leaves=4)) , 
('xgb', xgb.XGBRegressor(learning_rate=0.01, max_depth=4, n_estimators=4000))
]
def keyword_disaster_probabilities(x):
tweets_w_keyword = np.sum(train["keyword"].fillna("" ).str.contains(x))
tweets_w_keyword_disaster = np.sum(train["keyword"].fillna("" ).str.contains(x)& train["target"] == 1)
 return tweets_w_keyword_disaster / tweets_w_keyword
keywords_vc["Disaster_Probability"] = keywords_vc.index.map(keyword_disaster_probabilities)
 keywords_vc.head()<sort_values>
s_reg = e.StackingRegressor(
 estimators=estimators2,
cv=20,
n_jobs=-1
)
keywords_vc.sort_values(by="Disaster_Probability", ascending=False ).head(10 )<count_values>
X_train, X_val, y_train, y_val = ms.train_test_split(X, y, test_size=0.2 )
len(train["location"].value_counts() )<remove_duplicates>
s_reg.fit(X_train, y_train )
def create_corpus(target):
corpus = []
for w in train.loc[train["target"] == target]["text"].str.split() :
for i in w:
corpus.append(i)
 return corpus
def create_corpus_dict(target):
corpus = create_corpus(target)
 stop_dict = defaultdict(int)
 for word in corpus:
if word in stopwords:
stop_dict[word] += 1
return sorted(stop_dict.items() , key=lambda x:x[1], reverse=True )<count_duplicates>
s_reg.score(X_val, y_val )
corpus_disaster, corpus_non_disaster = create_corpus(1), create_corpus(0)
 counter_disaster, counter_non_disaster = Counter(corpus_disaster), Counter(corpus_non_disaster)
 x_disaster, y_disaster, x_non_disaster, y_non_disaster = [], [], [], []
counter = 0
for word, count in counter_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_disaster.append(word)
 y_disaster.append(count)
 counter = 0
for word, count in counter_non_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_non_disaster.append(word)
 y_non_disaster.append(count)
 fig, ax = plt.subplots(1,2, figsize=(20,8))
sns.barplot(x=y_disaster, y=x_disaster, orient='h', palette="Reds_d", ax=ax[0])
 sns.barplot(x=y_non_disaster, y=x_non_disaster, orient='h', palette="Blues_d", ax=ax[1])
 ax[0].set_title("Top 15 Non-Stopwords - Disaster Tweets")
 ax[0].set_xlabel("Word Frequency")
 ax[1].set_title("Top 15 Non-Stopwords - Non-Disaster Tweets")
 ax[1].set_xlabel("Word Frequency")
 plt.tight_layout()
plt.show()<feature_engineering>
s_reg.fit(X, y )
def bigrams(target):
corpus = train[train["target"] == target]["text"]
count_vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = count_vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in count_vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq<feature_engineering>
hp_pred = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
def remove_pattern(input_txt, pattern):
r = re.findall(pattern, input_txt)
 for i in r:
input_txt = re.sub(i, '', input_txt)
 return input_txt
train['tweet'] = np.vectorize(remove_pattern )(train['text'], "
test['tweet'] = np.vectorize(remove_pattern )(test['text'], "
train.head()
train['tweet'] = train['tweet'].str.replace("[^a-zA-Z
test['tweet'] = test['tweet'].str.replace("[^a-zA-Z
train.head()
train['tweet'] = train['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
test['tweet'] = test['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
train.head()
train['tweet'] = train['tweet'].str.lower()
test['tweet'] = test['tweet'].str.lower()
set(stopwords.words('english'))
stops = set(stopwords.words('english'))
train['tokenized_sents'] = train.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 test['tokenized_sents'] = test.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 def remove_stops(row):
my_list = row['tokenized_sents']
meaningful_words = [w for w in my_list if not w in stops]
return(meaningful_words)
 train['clean_tweet'] = train.apply(remove_stops, axis=1)
 test['clean_tweet'] = test.apply(remove_stops, axis=1)
 train.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 test.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 def rejoin_words(row):
my_list = row['clean_tweet']
joined_words =(" ".join(my_list))
return joined_words
train['clean_tweet'] = train.apply(rejoin_words, axis=1)
 test['clean_tweet'] = test.apply(rejoin_words, axis=1)
 train.head() 
<set_options>
X_pred = hp_pred.drop(columns=unused_features )
warnings.filterwarnings("ignore")
 tqdm.pandas()
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
 random_state = 29<set_options>
X_pred.isna().sum().sum()
!pip install GPUtil
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()<load_from_csv>
X_pred[numerical_features] = numerical_imputer.transform(X_pred[numerical_features] )
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sub= pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<define_variables>
X_pred[date_features] = date_imputer.transform(X_pred[date_features] )
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}<string_transform>
X_pred[categorical_features_to_impute_mode] = categorical_mode_imputer.transform(X_pred[categorical_features_to_impute_mode])
 X_pred[categorical_features_to_impute_none] = categorical_none_imputer.transform(X_pred[categorical_features_to_impute_none] )
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'URL',text)
 def remove_HTML(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_not_ASCII(text):
text = ''.join([word for word in text if word in string.printable])
 return text
def word_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def replace_abbrev(text):
string = ""
for word in text.split() :
string += word_abbrev(word)+ " "
return string
def remove_mention(text):
at=re.compile(r'@\S+')
 return at.sub(r'USER',text)
 def remove_number(text):
num = re.compile(r'[-+]?[.\d]*[\d]+[:,.\d]*')
 return num.sub(r'NUMBER', text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'EMOJI', text)
 def transcription_sad(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[(\\/]')
 return smiley.sub(r'SADFACE', text)
 def transcription_smile(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[)dDp]')
 return smiley.sub(r'SMILE', text)
 def transcription_heart(text):
heart = re.compile(r'<3')
 return heart.sub(r'HEART', text)
 def remove_elongated_words(text):
rep = re.compile(r'\b(\S*? )([a-z])\2{2,}\b')
 return rep.sub(r'\1\2 ELONG', text)
 def remove_repeat_punct(text):
rep = re.compile(r'([!?.]){2,}')
 return rep.sub(r'\1 REPEAT', text)
 def remove_all_punct(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_punct(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
def remove_stopwords(text):
text = ' '.join([word for word in text.split() if word not in(stopwords)])
 return text<drop_column>
X_pred[['LotFrontage', 'LotArea', '1stFlrSF', 'GrLivArea']] = np.log(X_pred[['LotFrontage', 'LotArea', '1stFlrSF', 'GrLivArea']] )
def clean_tweet(text):
text = remove_URL(text)
 text = remove_HTML(text)
 text = remove_not_ASCII(text)
 text = text.lower()
text = replace_abbrev(text)
 text = remove_mention(text)
 text = remove_number(text)
 text = remove_emoji(text)
 text = transcription_sad(text)
 text = transcription_smile(text)
 text = transcription_heart(text)
 text = remove_elongated_words(text)
 text = remove_repeat_punct(text)
 return text<feature_engineering>
create_features(X_pred )
train["clean_text"] = train["text"].apply(clean_tweet)
 test["clean_text"] = test["text"].apply(clean_tweet)
 train["clean_tokens"] = train["clean_text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["clean_text"].apply(lambda x: word_tokenize(x))<feature_engineering>
X_pred_oh = pd.DataFrame(one_hot_encoder.transform(X_pred[categorical_oh_features]), columns=one_hot_encoder.get_feature_names(categorical_oh_features))
X_pred_oh
skip_gram_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2,sg=1)
 skip_gram_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 cbow_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2)
 cbow_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 train["clean_tokens"] = train["text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["text"].apply(lambda x: word_tokenize(x))
def get_word_embeddings(token_list,vector,k=150):
if len(token_list)< 1:
return np.zeros(k)
 else:
vectorized = [vector[word] if word in vector else np.random.rand(k)for word in token_list]
sum = np.sum(vectorized,axis=0)
 return sum/len(vectorized )<feature_engineering>
X_pred_oh.index = X_pred.index
X_pred.drop(categorical_oh_features, axis=1, inplace=True)
 X_pred = pd.concat([X_pred, X_pred_oh], axis=1)
 X_pred
max_features=5000
count_vectorizer = CountVectorizer(max_features=max_features)
 sparce_matrix_train=count_vectorizer.fit_transform(train['clean_text'])
 sparce_matrix_test=count_vectorizer.fit_transform(train['clean_text'])
 def count_vector(data):
count_vectorizer = CountVectorizer()
vect = count_vectorizer.fit_transform(data)
 return vect, count_vectorizer
def tfidf_vector(data):
tfidf_vectorizer = TfidfVectorizer()
vect = tfidf_vectorizer.fit_transform(data)
 return vect, tfidf_vectorizer
def skip_gram_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, skip_gram_model))
return list(embeddings)
 def cbow_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, cbow_model))
return list(embeddings)
 X_train_count, count_vectorizer = count_vector(train['clean_text'])
 X_train_tfidf, tfidf_vectorizer = tfidf_vector(train['clean_text'])
 X_train_skip_gram = skip_gram_vectorizer(train['clean_tokens'],skip_gram_model)
 X_train_cbow = cbow_vectorizer(train['clean_tokens'],cbow_model)
 X_test_count = count_vectorizer.transform(test['clean_text'])
 X_test_tfidf = tfidf_vectorizer.transform(test['clean_text'])
 X_test_skip_gram = skip_gram_vectorizer(test['clean_tokens'],skip_gram_model)
 X_test_cbow = cbow_vectorizer(test['clean_tokens'],cbow_model)
 X_train_skip_gram = np.array(X_train_skip_gram)
 X_train_cbow = np.array(X_train_cbow)
 X_test_skip_gram = np.array(X_test_skip_gram)
 X_test_cbow = np.array(X_test_cbow)
 with open('count_vectorizer.pickle', 'wb')as handle:
pickle.dump(count_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('tfidf_vectorizer.pickle', 'wb')as handle:
pickle.dump(tfidf_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('skip_gram_model.pickle', 'wb')as handle:
pickle.dump(skip_gram_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
<create_dataframe>
X_pred[categorical_l_features] = pd.DataFrame(ordinal_encoder.transform(X_pred[categorical_l_features]), columns=X_pred[categorical_l_features].columns)
 X_pred[categorical_l_features]
metrics = pd.DataFrame(columns=['model' ,'vectoriser', 'f1 score', 'train accuracy','test accuracy'] )<choose_model_class>
X_pred = pd.DataFrame(scaler.transform(X_pred),columns=X_pred.columns)
 X_pred
models=[
XGBClassifier(max_depth=6, n_estimators=1000),
LogisticRegression(random_state=random_state),
SVC(random_state=random_state),
MultinomialNB() ,
DecisionTreeClassifier(random_state = random_state),
KNeighborsClassifier() ,
RandomForestClassifier(random_state=random_state),
]<split>
for model in models:
y = train.target
x = X_train_count
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test,'Count vector')
 x = X_train_tfidf
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'Tfidf vector')
 x = X_train_skip_gram
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'skip gram vector')
 x = X_train_cbow
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'cbow vector' )<sort_values>
y_pred = pd.DataFrame()
y_pred['Id'] = hp_pred.Id
metrics = metrics.sort_values('f1 score',ascending=False )<find_best_params>
y_pred['SalePrice'] = np.exp(s_reg.predict(X_pred))
free_gpu_cache()<import_modules>
y_pred.to_csv('submission.csv', index=False )