Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
import pandas as pd
import numpy as np
import re
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<drop_column>
def htmlremove(text):
return re.sub('<[^<]+?>', '', text)
 train['text'] = train['text'].apply(htmlremove)
 test['text'] = test['text'].apply(htmlremove )
test_id = test['id']
del train['id']
del test['id']<categorify>
def urlremove(text):
return re.sub(r"(?:\@|https?\://)\S+", "", text)
 train['text'] = train['text'].apply(urlremove)
 test['text'] = test['text'].apply(urlremove )
encoder = OneHotEncoder(cols=['color'], use_cat_names=True)
 train = encoder.fit_transform(train)
 test = encoder.fit_transform(test )<categorify>
def remove_emoji(text):
emoji_pattern = re.compile(
 u"(\ud83d[\ude00-\ude4f])|"
u"(\ud83c[\udf00-\uffff])|"
u"(\ud83d[\u0000-\uddff])|"
u"(\ud83d[\ude80-\udeff])|"
u"(\ud83c[\udde0-\uddff])"
"+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train['text'] = train['text'].apply(remove_emoji)
 test['text'] = test['text'].apply(remove_emoji )
encoder = LabelEncoder()
encoder.fit(train['type'])
 print(encoder.classes_)
 train['type_no'] = encoder.transform(train['type'] )<prepare_x_and_y>
def decontracted(text):
if '’' in text:
text = text.replace("’", "'")
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"can't", "can not", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 return text
train['text'] = train['text'].apply(decontracted)
 test['text'] = test['text'].apply(decontracted )
target = train['type_no']
target_string = train['type']
del train['type']
del train['type_no']
target.head()<split>
def characterremove(text):
return re.sub('\W+|_', ' ', text)
 train.text = train.text.apply(characterremove)
 test.text = test.text.apply(characterremove )
train_data, test_data, train_target, test_target = train_test_split(train, target, test_size=0.2, random_state=42 )<import_modules>
def remove_numbers(text):
pattern = r'[^a-zA-z.,!?/:;"'\s]'
return re.sub(pattern, '', text)
 train.text = train.text.apply(remove_numbers)
 train.text = train.text.str.lower()
train.text = train.text.str.strip()
test.text = test.text.apply(remove_numbers)
 test.text = test.text.str.lower()
test.text = test.text.str.strip()
decisions(DecisionTreeClassifier() , ['hair_length', 'bone_length'] )<set_options>
stopwords = nltk.corpus.stopwords.words('english')
 def get_top_20(ngram_range, new_corpus, stopwords, min_df):
init_vec = CountVectorizer(ngram_range=ngram_range, stop_words = stopwords, min_df=min_df)
 X = init_vec.fit_transform(new_corpus)
 count_list = X.toarray().sum(axis=0)
 word_list = init_vec.get_feature_names()
counts = dict(zip(word_list,count_list))
top_20 = sorted(counts.items() , key=lambda x:x[1], reverse=True)[:20]
return top_20, counts
top_20, counts = get_top_20(( 1,1), train.text, stopwords, 4)
 top_20
Image(url= 'https://cdn-images-1.medium.com/max/1600/1*JZbxrdzabrT33Yl-LrmShw.png', width=750, height=750 )<train_model>
lemmer = WordNetLemmatizer()
train.text = [' '.join([lemmer.lemmatize(word)for word in text.split(' ')])for text in train.text]
test.text = [' '.join([lemmer.lemmatize(word)for word in text.split(' ')])for text in test.text]
clfs = [RandomForestClassifier() , AdaBoostClassifier() , SVC() , KNeighborsClassifier() ]
labels = ['Random Forest', 'Ada Boost', 'Support Vector', 'K-Neighbors']
for model, name in zip(clfs, labels):
model.fit(train_data, train_target)
 predictions = model.predict(test_data)
 print('{} accuracy is: {}'.format(name, accuracy_score(test_target, predictions)) )<train_on_grid>
X_text = train.text
y_text = train.target
X_test_text = test.text
params = {
'C': [0.001, 0.01, 0.1, 1, 10],
'gamma': [0.001, 0.01, 0.1, 1]
}
grid_search = GridSearchCV(SVC() , params, cv=5)
 grid_search.fit(train, target)
 grid_search.best_params_<train_model>
nlp = spacy.load("en_core_web_lg")
 nlp.vocab["amp"].is_stop = True
docs = [nlp(d ).vector for d in X_text]
X_text = np.vstack(docs)
 docs_test = [nlp(d ).vector for d in X_test_text]
X_test_text = np.vstack(docs_test )
importance_model = SVC(C=10, gamma=0.1, probability=True)
 importance_model.fit(train, target)
 perm = PermutationImportance(importance_model, random_state=42 ).fit(test_data, test_target)
 eli5.show_weights(perm, feature_names=test_data.columns.tolist() )<filter>
X_train, X_test, y_train, y_test = train_test_split(X_text, y_text, random_state=0 )
test_target.iloc[0], target_string.iloc[0]<train_model>
param_grid = {'kernel': ['rbf','linear'], 'C': [0.001, 0.01, 0.1, 1, 1.5, 2, 3, 10]}
grid = GridSearchCV(SVC() , param_grid, cv=10)
 grid.fit(X_train, y_train)
 print(grid.best_params_)
 grid.score(X_test,y_test )
<predict_on_test><EOS> 
y_test_text = grid.predict(X_test_text)
 test['target'] = y_test_text
test_export = test.loc[:, ['id', 'target']]
test_export.to_csv('disaster_test_word2vec.csv', index=False )