Kaggle competition: compass-canada
<SOS> metric: rmse Kaggle data source: compass-canada<rename_columns>
import numpy as np
import pandas as pd
import os
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()<save_to_csv>
train_dataset_full = pd.read_csv('.. /input/08_train_dataset.csv' )
submission_df.to_csv("submission.csv", index=False, header=True )<import_modules>
test_dataset = pd.read_csv('.. /input/08_test_dataset_no_response.csv' )
import tensorflow as tf
import operator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import *
import pandas as pd
import numpy as np
from gensim.models import Word2Vec, KeyedVectors
import gensim
import time
import random
import matplotlib.pyplot as plt
import string
import nltk
from tensorflow.keras.preprocessing.sequence import pad_sequences
import multiprocessing
import nltk
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk import word_tokenize, pos_tag
import functools
from gensim.models.fasttext import FastText
from nltk.corpus import wordnet
import re
from nltk.tokenize.treebank import TreebankWordDetokenizer
from wordsegment import load, segment<load_from_csv>
import seaborn as sns
disaster_text = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv", encoding='utf-8' )<data_type_conversions>
import tensorflow as tf
disaster_text['keyword'] = disaster_text['keyword'].astype(str)
 disaster_text.loc[disaster_text['keyword'] == 'nan','keyword'] = ''<data_type_conversions>
LABEL='volume_in_cans'
FEATURE_COLUMNS = ['vpo_stores_nearby','closest_store_vpo','closest_store_dist','stores_count_nearby',
'population_2018_year_average_total_number','population_2018_year_average_per_mill_of_country',
'purchasing_power_2018_million_euro','purchasing_power_2018_per_mill_of_country','purchasing_power_2018_euro_per_capita',
'purchasing_power_2018_index_country_eq_100']
disaster_text['location'] = disaster_text['location'].astype(str)
 disaster_text.loc[disaster_text['location'] == 'nan','location'] = ''<count_unique_values>
def make_train_input_fn(df, num_epochs):
return tf.estimator.inputs.pandas_input_fn(
 x = df[FEATURE_COLUMNS],
y = df[LABEL],
batch_size = 256,
num_epochs = num_epochs,
shuffle = True,
queue_capacity = 1000
)
df_mislabeled = disaster_text.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
df_mislabeled.index.tolist()<feature_engineering>
def make_eval_input_fn(df):
return tf.estimator.inputs.pandas_input_fn(
 x = df[FEATURE_COLUMNS],
y = df[LABEL],
batch_size = 256,
shuffle = False,
queue_capacity = 1000
)
disaster_text['target_relabeled'] = disaster_text['target'].copy()
disaster_text.loc[disaster_text['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you!
disaster_text.loc[disaster_text['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == '.POTUS
disaster_text.loc[disaster_text['text'] == 'CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1
disaster_text.loc[disaster_text['text'] == '
disaster_text.loc[disaster_text['text'] == 'In
disaster_text.loc[disaster_text['text'] == 'Who is bringing the tornadoes and floods.Who is bringing the climate change.God is after America He is plaguing her
disaster_text.loc[disaster_text['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B.Cooper.http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1
disaster_text.loc[disaster_text['text'] == "Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "wowo--=== 12000 Nigerian refugees repatriated from Cameroon", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "He came to a land which was engulfed in tribal war and turned it into a land of peace i.e.Madinah.
 disaster_text.loc[disaster_text['text'] == "Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it
disaster_text.loc[disaster_text['text'] == "The Prophet(peace be upon him)said 'Save yourself from Hellfire even if it is by giving half a date in charity.'", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "Caution: breathing may be hazardous to your health.", 'target_relabeled'] = 1
disaster_text.loc[disaster_text['text'] == "I Pledge Allegiance To The P.O.P.E.And The Burning Buildings of Epic City.??????", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "
disaster_text.loc[disaster_text['text'] == "that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time", 'target_relabeled'] = 0<feature_engineering>
def make_prediction_input_fn(df):
return tf.estimator.inputs.pandas_input_fn(
 x = df[FEATURE_COLUMNS],
y = None,
batch_size = 128,
shuffle = False,
queue_capacity = 1000
)
disaster_text['target'] = disaster_text['target_relabeled']<define_search_model>
def make_feature_cols() :
input_columns = [tf.feature_column.numeric_column(k)for k in FEATURE_COLUMNS]
return input_columns
def to_lower(x):
x = x.lower()
return(x)
 def tokenization(x):
x = word_tokenize(x)
 return(x)
 def get_pos(word):
tag = nltk.pos_tag([word])[0][1][0].upper()
tag_dict = {"J": wordnet.ADJ,
"N": wordnet.NOUN,
"V": wordnet.VERB,
"R": wordnet.ADV}
return tag_dict.get(tag, wordnet.NOUN)
 def stemming_and_lemmatization(x):
lemmatizer = WordNetLemmatizer()
y = [lemmatizer.lemmatize(w, get_pos(w)) for w in x]
return(y)
 def remove_stopwords(x):
return([word for word in x if word not in stopwords.words('english')])
 def remove_punctuations(x):
x = re.sub(re.compile('(( http|ftp|https):\/\/)?[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+
x = re.sub('[^\w\s]','',x)
 x = re.sub("[^a-zA-Z\s]+", '', x)
 return(x)
 def decontracted(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
def hashtags(phrase):
phrase = " ".join(segment(phrase))
return phrase
def spellCheck(phrase):
spell = Speller(lang='en')
 phrase = "".join([spell(w)+" " for w in phrase.split() ])
 return phrase
def text_preprocessing(x):
x = to_lower(x)
 x = decontracted(x)
 x = remove_punctuations(x)
 return(x )<feature_engineering>
import shutil
p_time1 = time.time()
count = 0
load()
disaster_text['preprocessed_text'] = disaster_text['text'].apply(text_preprocessing)
 p_time2 = time.time()
disaster_text.head(10 )<define_variables>
tf.logging.set_verbosity(tf.logging.INFO)
 OUTDIR = './test_model'
shutil.rmtree(OUTDIR, ignore_errors = True)
 model = tf.estimator.LinearRegressor(
 feature_columns = make_feature_cols() , model_dir = OUTDIR)
 model.train(input_fn = make_train_input_fn(train_dataset_full, num_epochs = 1))
p_time2 - p_time1<load_pretrained>
def print_rmse(model, df):
metrics = model.evaluate(input_fn = make_eval_input_fn(df))
print('RMSE on dataset = {}'.format(np.sqrt(metrics['average_loss'])) )
glove2word2vec(glove_input_file="/kaggle/input/glove-pretrained/glove.6B.300d.txt", word2vec_output_file="gensim_glove_vectors.txt" )<feature_engineering>
print_rmse(model, train_dataset_full )
cores = multiprocessing.cpu_count()
w_time1 = time.time()
word2VecModel = gensim.models.KeyedVectors.load_word2vec_format('gensim_glove_vectors.txt', binary=False)
 w_time2 = time.time()<define_variables>
predictions = model.predict(input_fn = make_prediction_input_fn(test_dataset))
predicted_vol = [item['predictions'][0] for item in predictions]
w_time2 - w_time1<string_transform>
test_dataset['lr_test_preds'] = predicted_vol
<count_unique_values><EOS> 
test_dataset[['google_place_id','lr_test_preds']].to_csv(f"submission-{int(time())}.csv", index=False )