Kaggle competition: Home Depot Product Search Relevance
<SOS> metric: RMSE Kaggle data source: home-depot-product-search-relevance<define_variables>
!unzip.. /input/home-depot-product-search-relevance/attributes.csv.zip
!unzip.. /input/home-depot-product-search-relevance/product_descriptions.csv.zip
!unzip.. /input/home-depot-product-search-relevance/test.csv.zip
!unzip.. /input/home-depot-product-search-relevance/train.csv.zip
np.random.seed(42 )<load_from_csv>
!pip install bs4
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )<concatenate>
import pandas as pd
import re
import math
from nltk.stem.snowball import SnowballStemmer
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
from bs4 import BeautifulSoup
import lxml
import re
import nltk
from nltk.corpus import stopwords
from nltk.metrics import edit_distance
from string import punctuation
from collections import Counter
import numpy as np
all_data = pd.concat([train, test], ignore_index=True )<count_values>
train_data = pd.read_csv("train.csv", encoding="ISO-8859-1")
 test_data = pd.read_csv("test.csv", encoding="ISO-8859-1")
 attribute_data = pd.read_csv('attributes.csv')
 descriptions = pd.read_csv('product_descriptions.csv' )
def null_value_counts(df, col, null=True):
if null:
return len(df[df[col].isnull() ][col].to_list())
 else:
return len(df[~df[col].isnull() ][col].to_list())
<feature_engineering>
train_data[train_data.duplicated(keep='first')].shape
def punctuations_checker(texts, get_top_15=True):
punctuations = ['\' + p for p in list(string.punctuation)if p not in list(',.?"'@
dirty_words = defaultdict()
for text in texts:
for word in text.split() :
if all([(len(re.findall(r'|'.join(punctuations), word)) > 1),(len(word)> 1)]):
try:
dirty_words[word] += 1
except:
dirty_words[word] = 1
if get_top_15:
dirty_words = {k: v for k, v in sorted(dirty_words.items() , key=lambda item: item[1], reverse=True)[:15]}
else:
dirty_words = {k: v for k, v in sorted(dirty_words.items() , key=lambda item: item[1], reverse=True)}
return dirty_words 
<string_transform>
test_data[test_data.duplicated(keep='first')].shape
dirty_words_real_disaster = punctuations_checker(all_data.text, get_top_15=True)
<filter>
attribute_data[attribute_data.duplicated(keep='first')]
attribute_data = attribute_data.drop_duplicates()
print(all_data[all_data.text.str.contains(';')].text.to_list() [:30] )<categorify>
descriptions[descriptions.duplicated(keep='first')].shape
def preprocess(texts):
clean_text = []
for text in texts:
text = html.unescape(text)
 text = ftfy.fix_text(text)
 text = re.sub(r'(?:http(?:s)?://)?\bt\b\.\bco\b(?:/[\w/-]+)?', ' ', text)
 text = re.sub(r'&gt;', '>', text)
 text = re.sub(r'&lt;', '<', text)
 text = re.sub(r'&amp;', '&', text)
 text = re.sub('\\
', '
', text)
 text = re.sub(r'\x89Û', '', text)
 text = re.sub(r'‰|_|Ï|Ò|ª|÷|å|©|£|À|Ì|Û|Ê', ' ', text)
 text = text.replace('...', '')
 text = text.replace('%20', ' ')
 for p in list(string.punctuation):
text = text.replace(p, f' {p} ')
 text = re.sub('\s+', ' ', text ).strip().lower()
clean_text.append(text)
 return clean_text<categorify>
print('total data has html tags in',descriptions.product_description.str.count('<br$' ).values.sum() )
all_data['text'] = preprocess(all_data.text)
<string_transform>
df_brand = attribute_data[attribute_data.name == "MFG Brand Name"][["product_uid", "value"]].rename(columns={"value": "brand"} )
def words_generator(texts, n_grams=2, get_top_50=True):
gram2idx = defaultdict()
for line in texts:
word_list = line.split(' ')
 filtered_words = [word.strip() for word in word_list if word not in stopwords.words('english')]
grams = list(ngrams(filtered_words, n_grams))
grams_merge = []
for g in grams:
g = ' '.join(map(str,g))
grams_merge.append(g)
 grams = grams_merge
for g in grams:
if len(g.strip())> 3 and any([char_g.isalpha() for char_g in set(g.strip())]):
try:
gram2idx[g] += 1
except:
gram2idx[g] = 1
if get_top_50:
gram2idx = {k: v for k, v in sorted(gram2idx.items() , key=lambda item: item[1], reverse=True)[:50]}
else:
gram2idx = {k: v for k, v in sorted(gram2idx.items() , key=lambda item: item[1], reverse=True)}
return gram2idx<string_transform>
df_attr_stripped = attribute_data
df_attr_stripped['name'] = df_attr_stripped['name'].astype(str)
 df_attr_stripped['name'] = df_attr_stripped['name'].apply(lambda s: re.sub(r"Bullet([0-9]+)", "", s))
df_attr_stripped['attribute'] = df_attr_stripped['name'] + " " + df_attr_stripped['value']
df_attr_test = df_attr_stripped.groupby('product_uid' ).agg({'attribute': lambda s : ' '.join(s.astype(str)) } ).reset_index()
realdis_trigram_count = words_generator(all_data[all_data.target == 1].text[::], n_grams=3, get_top_50=True)
 notdis_trigram_count = words_generator(all_data[all_data.target == 0].text[::], n_grams=3, get_top_50=True)
 realdis_bigram_count = words_generator(all_data[all_data.target == 1].text[::], n_grams=2, get_top_50=True)
 notdis_bigram_count = words_generator(all_data[all_data.target == 0].text[::], n_grams=2, get_top_50=True)
 realdis_words_count = words_generator(all_data[all_data.target == 1].text[::], n_grams=1, get_top_50=True)
 notdis_words_count = words_generator(all_data[all_data.target == 0].text[::], n_grams=1, get_top_50=True )<import_modules>
data_all = pd.concat([train_data, test_data], axis=0, ignore_index=True )
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator<string_transform>
data_all = pd.merge(data_all, descriptions, how = 'left', on = 'product_uid' )
words_real_disaster = ' '.join(all_data[all_data.target == 1].text.to_list())
 words_not_disaster = ' '.join(all_data[all_data.target == 0].text.to_list() )<import_modules>
data_all = pd.merge(data_all, df_brand, how = 'left', on = 'product_uid' )
from sklearn.feature_extraction.text import CountVectorizer,TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split<split>
data_all = pd.merge(data_all, df_attr_test, how = 'left', on = 'product_uid' )
train['clean_text'] = preprocess(train.text)
 train = train.drop_duplicates(subset='clean_text', keep='first')
 print('length train data', len(train))
print('split 80:20')
 X = train.clean_text
y = train.target
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
 train_0, train_1 = len(y_train[y_train==0]), len(y_train[y_train==1])
 test_0, test_1 = len(y_val[y_val==0]), len(y_val[y_val==1])
 print('Train: 0=%d, 1=%d, Test: 0=%d, 1=%d' %(train_0, train_1, test_0, test_1))
cat_0 = train_0 + test_0
cat_1 = train_1 + test_1
print('Train: 0=%f, 1=%f '%(train_0/cat_0, train_1/cat_1))
print('Test: 0=%f, 1=%f'%(test_0/cat_0, test_1/cat_1))<import_modules>
def string_edit(s):
if isinstance(s, str):
s = re.sub(r"(\w)\.( [A-Z])", r"\1 \2", s)
 s = s.lower()
s = s.replace(" "," ")
 s = s.replace(",","")
 s = s.replace("$"," ")
 s = s.replace("?"," ")
 s = s.replace("-"," ")
 s = s.replace("//","/")
 s = s.replace(".. ",".")
 s = s.replace(" / "," ")
 s = s.replace(" \\ "," ")
 s = s.replace(".",".")
 s = re.sub(r"(^\.|/)", r"", s)
 s = re.sub(r"(\.|/)$", r"", s)
 s = re.sub(r"([0-9] )([a-z])", r"\1 \2", s)
 s = re.sub(r"([a-z] )([0-9])", r"\1 \2", s)
 s = s.replace(" x "," xbi ")
 s = re.sub(r"([a-z] )(*)\.( * )([a-z])", r"\1 \4", s)
 s = re.sub(r"([a-z] )(*)/(* )([a-z])", r"\1 \4", s)
 s = s.replace("*"," xbi ")
 s = s.replace(" by "," xbi ")
 s = re.sub(r"([0-9] )(*)\.( * )([0-9])", r"\1.\4", s)
 s = re.sub(r"([0-9]+ )(* )(inches|inch|in|')\.?", r"\1in.", s)
 s = re.sub(r"([0-9]+ )(* )(foot|feet|ft|'')\.?", r"\1ft.", s)
 s = re.sub(r"([0-9]+ )(* )(pounds|pound|lbs|lb)\.?", r"\1lb.", s)
 s = re.sub(r"([0-9]+ )(* )(square|sq)?\.?(feet|foot|ft)\.?", r"\1sq.ft.", s)
 s = re.sub(r"([0-9]+ )(* )(cubic|cu)?\.?(feet|foot|ft)\.?", r"\1cu.ft.", s)
 s = re.sub(r"([0-9]+ )(* )(gallons|gallon|gal)\.?", r"\1gal.", s)
 s = re.sub(r"([0-9]+ )(* )(ounces|ounce|oz)\.?", r"\1oz.", s)
 s = re.sub(r"([0-9]+ )(* )(centimeters|cm)\.?", r"\1cm.", s)
 s = re.sub(r"([0-9]+ )(* )(milimeters|mm)\.?", r"\1mm.", s)
 s = s.replace("°"," degrees ")
 s = re.sub(r"([0-9]+ )(* )(degrees|degree)\.?", r"\1deg.", s)
 s = s.replace(" v "," volts ")
 s = re.sub(r"([0-9]+ )(* )(volts|volt)\.?", r"\1volt.", s)
 s = re.sub(r"([0-9]+ )(* )(watts|watt)\.?", r"\1watt.", s)
 s = re.sub(r"([0-9]+ )(* )(amperes|ampere|amps|amp)\.?", r"\1amp.", s)
 s = s.replace(" "," ")
 s = s.replace("."," ")
 s = re.sub(r"zero\.?", r"0 ", s)
 s = re.sub(r"one\.?", r"1 ", s)
 s = re.sub(r"two\.?", r"2 ", s)
 s = re.sub(r"three\.?", r"3 ", s)
 s = re.sub(r"four\.?", r"4 ", s)
 s = re.sub(r"five\.?", r"5 ", s)
 s = re.sub(r"six\.?", r"6 ", s)
 s = re.sub(r"seven\.?", r"7 ", s)
 s = re.sub(r"eight\.?", r"8 ", s)
 s = re.sub(r"nine\.?", r"9 ", s)
 return s
else:
return "null"
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.linear_model import SGDClassifier, LogisticRegression, RidgeClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.preprocessing import FunctionTransformer<define_variables>
data_all['search_term'] = data_all['search_term'].map(lambda x:string_edit(str(x)))
 data_all['product_title'] = data_all['product_title'].map(lambda x:string_edit(str(x)))
 data_all['product_description'] = data_all['product_description'].map(lambda x:string_edit(str(x)) )
np.random.seed(42 )<choose_model_class>
data_all['attribute'] = data_all['attribute'].map(lambda x:string_edit(str(x)))
 data_all['brand'] = data_all['brand'].map(lambda x:string_edit(str(x)) )
model_sgd = SGDClassifier()
model_lsvm = LinearSVC()
model_rf = RandomForestClassifier()
model_lr = LogisticRegression()
model_r = RidgeClassifier()
eclf = VotingClassifier(estimators=[('lr', model_lr),('sgd', model_sgd),('lsvm', model_lsvm)], voting='hard')
 for model, label in zip([model_sgd, model_lsvm, model_rf, model_r, model_lr, eclf],
['SGD', 'LinearSVC', 'RandomForest', 'RidgeClassifier', 'LogisticRegression', 'VotingEnsemble']):
clf = Pipeline([ 
('vect', CountVectorizer(ngram_range=(1,1))), 
('tfidf', TfidfTransformer()), 
('to_dense', FunctionTransformer(lambda x: x.toarray() , accept_sparse=True)) , 
('model', model)
 ])
 clf.fit(X_train,y_train)
 print("Acc: %f using" % clf.score(X_val, y_val), f'[{label}]' )<compute_test_metric>
def remove_html_tag(text):
soup = BeautifulSoup(text, 'lxml')
 text = soup.get_text().replace('Click here to review our return policy for additional information regarding returns', '')
 return text
def str_stemmer(doc):
tokens = doc.split()
table = str.maketrans('', '', punctuation)
 tokens = [w.translate(table)for w in tokens]
tokens = [word for word in tokens if word.isalpha() ]
stop_words = set(stopwords.words('english'))
tokens = [w for w in tokens if not w in stop_words]
tokens = [word for word in tokens if len(word)> 1]
return ' '.join(tokens)
 def str_stemmer_title(s):
return " ".join(map(stemmer.stem, s.lower().split()))
def str_common_word(str1, str2):
whole_set = set(str1.split())
 return sum(int(str2.find(word)>=0)for word in whole_set)
 def get_shared_words(row_data):
return np.sum([str_common_word(*row_data[:-1]), str_common_word(*row_data[1:])] )
train_acc = clf.score(X_train, y_train)
 val_acc = clf.score(X_val, y_val)
 print('train_acc:', train_acc)
 print('val_acc:', val_acc )<load_pretrained>
data_all['search_term'] = pd.Series(data_all['search_term'].map(lambda x:str_stemmer(str(x))))
data_all['product_title'] = pd.Series(data_all['product_title'].map(lambda x:str_stemmer(str(x))))
data_all['product_description'] = pd.Series(data_all['product_description'].map(lambda x:str_stemmer(str(x))))
data_all['attribute'] = pd.Series(data_all['attribute'].map(lambda x:str_stemmer(str(x))))
!wget -O "/kaggle/working/glove.twitter.27B.zip" "http://nlp.stanford.edu/data/glove.twitter.27B.zip"<load_from_disk>
data_all['search_term'] = pd.Series(data_all['search_term'].map(lambda x:remove_html_tag(str(x))))
data_all['product_title'] = pd.Series(data_all['product_title'].map(lambda x:remove_html_tag(str(x))))
data_all['product_description'] = pd.Series(data_all['product_description'].map(lambda x:remove_html_tag(str(x))))
!unzip -j "/kaggle/working/glove.twitter.27B.zip" "glove.twitter.27B.200d.txt"<import_modules>
data_all['search_term_tokens'] = data_all.search_term.str.lower().str.split()
data_all['product_title_tokens'] = data_all.product_title.str.lower().str.split()
data_all['product_description_tokens'] = data_all.product_description.str.lower().str.split()
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import random<set_options>
data_all["edistance_sprot"] = [edit_distance(word1, word2)for word1, word2 in
data_all[["search_term","product_title"]].values.tolist() ]
data_all["edistance_sd"] = [edit_distance(word1, word2)for word1, word2 in
data_all[["search_term","product_description"]].values.tolist() ]
random.seed(360)
 np.random.seed(360)
 torch.manual_seed(360)
 torch.backends.cudnn.deterministic = True
torch.cuda.manual_seed_all(360 )<string_transform>
def get_jaccard_sim(columns):
str1, str2 = columns[0], columns[1]
a = set(str1)
 b = set(str2)
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
word2idx = {}
idx2word = {}
all_text = preprocess(train.text)+ preprocess(test.text)
 idx = 1
for text in all_text:
words = nltk.word_tokenize(text)
 for word in words:
if word not in word2idx.keys() :
word2idx[word] = idx
if idx not in idx2word.keys() :
idx2word[idx] = word
idx += 1<feature_engineering>
data_all['j_dis_sqt'] = [get_jaccard_sim(rows)for rows in data_all[["search_term_tokens","product_title_tokens"]].values]
data_all['j_dis_sqd'] = [get_jaccard_sim(rows)for rows in data_all[["search_term_tokens","product_description_tokens"]].values]
GLOVE_DIR = '/kaggle/working/'
embeddings_index = {}
f = open(os.path.join(GLOVE_DIR, 'glove.twitter.27B.200d.txt'))
for line in tqdm(f, desc='load glove embedding'):
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()<categorify>
data_all['search_query_length'] = data_all.search_term.str.len()
data_all['number_of_words_in_descr'] = data_all.product_description.str.count("\\w+" )
embedding_matrix = np.zeros(( len(word2idx)+ 2, 200))
for word, idx in tqdm(word2idx.items() , desc='build embedding matrix'):
try:
embedding_matrix[idx] = embeddings_index[word]
except:
embedding_matrix[-1] = np.random.rand(1, 200 )<categorify>
data_all['number_of_words_in_search'] = data_all['search_term_tokens'].map(lambda x: len(x))
data_all['number_of_words_in_title'] = data_all['product_title_tokens'].map(lambda x: len(x))
data_all['length_product_description'] = data_all.product_description.str.len()
data_all['length_product_title'] = data_all.product_title.str.len()
data_all['length_attribute'] = data_all.attribute.str.len()
def generate_data_loader(x, y, maxlen=128, with_target=True, bs=16):
all_data = []
for text, label in zip(x, y):
tokens = word_tokenize(text)
 token_ids = [word2idx[t] for t in tokens]
if len(token_ids)> maxlen:
token_ids = token_ids[:maxlen]
else:
token_ids = token_ids +([0] *(maxlen - len(token_ids)))
 token_ids_tensor = torch.LongTensor(token_ids)
 target = torch.tensor(label)
 data = {
'token_ids': token_ids_tensor,
'target' : target
}
all_data.append(data)
 data_loader = DataLoader(all_data, batch_size=bs, num_workers=5)
 return data_loader<load_pretrained>
data_all['len_of_attribute'] = data_all['attribute'].map(lambda x:len(str(x ).split())).astype(np.int64)
 data_all['len_of_brand'] = data_all['brand'].map(lambda x:len(str(x ).split())).astype(np.int64 )
train_dataloader = generate_data_loader(X_train, y_train)
 val_dataloader = generate_data_loader(X_val, y_val)
<find_best_params>
def cosineSim(v1, v2):
sumxx, sumxy, sumyy = 0, 0, 0
for i in range(len(v1)) :
x = v1[i]
y = v2[i]
sumxx += x*x
sumyy += y*y
sumxy += x*y
return sumxy/math.sqrt(sumxx*sumyy )
def evaluate(data_loader, criterion, model):
n_val_total, n_val_correct, val_loss_total = 0, 0, 0
model.eval()
with torch.no_grad() :
for sample_batched in data_loader:
inputs = sample_batched['token_ids'].to(device)
 outputs = model(inputs)
 targets = sample_batched['target'].to(device ).float().unsqueeze(1)
 loss = criterion(outputs, targets)
 n_val_correct += get_n_correct(targets, outputs)
 n_val_total += len(outputs)
 val_loss_total += loss.item() * len(outputs)
 val_acc = n_val_correct / n_val_total
val_loss = val_loss_total / n_val_total
return val_acc, val_loss 
<choose_model_class>
arr1 = []
arr2 = []
arr3 = []
for i in range(len(data_all)) :
product_title_i = data_all['product_title'][i]
description_i = data_all['product_description'][i]
attribute_i = data_all['attribute'][i]
search_term_i = data_all['search_term'][i]
TfidfVectorizer1 = TfidfVectorizer(ngram_range=(1, 2), stop_words='english')
 TfidfVectorizer2 = TfidfVectorizer(ngram_range=(1, 2), stop_words='english')
 TfidfVectorizer3 = TfidfVectorizer(ngram_range=(1, 2), stop_words='english')
 tf_idf1 = TfidfVectorizer1.fit_transform([product_title_i, search_term_i])
 tf_idf2 = TfidfVectorizer2.fit_transform([description_i, search_term_i])
 tf_idf3 = TfidfVectorizer3.fit_transform([attribute_i, search_term_i])
 cosineSim_title_search = cosineSim(tf_idf1.toarray() [0], tf_idf1.toarray() [1])
 arr1.append(cosineSim_title_search)
 cosineSim_description = cosineSim(tf_idf2.toarray() [0], tf_idf2.toarray() [1])
 arr2.append(cosineSim_description)
 cosineSim_attribute = cosineSim(tf_idf3.toarray() [0], tf_idf3.toarray() [1])
 arr3.append(cosineSim_attribute )
class LSTMModel(nn.Module):
def __init__(self, embedding_matrix, output_size=1, embedding_dim=200, hidden_dim=300, hidden_dim_2=100,
n_layers=1, drop_prob=0.5, bidirectional=False, pooling=False):
super().__init__()
self.output_size = output_size
self.n_layers = n_layers
self.hidden_dim = hidden_dim
self.embedding = nn.Embedding.from_pretrained(torch.tensor(embedding_matrix, dtype=torch.float))
self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, batch_first=True, bidirectional=bidirectional)
 self.dropout = nn.Dropout(drop_prob)
 self.pooling = pooling
self.bidirectional = bidirectional
self.fc_b_pool = nn.Linear(hidden_dim*4, hidden_dim*2)
 if self.bidirectional or self.pooling:
self.fc = nn.Linear(hidden_dim*2, hidden_dim)
 self.fc_out = nn.Linear(hidden_dim, output_size)
 else:
self.fc = nn.Linear(hidden_dim, hidden_dim_2)
 self.fc_out = nn.Linear(hidden_dim_2, output_size)
 def forward(self, x):
embeds = self.embedding(x)
 len_seq = torch.as_tensor(( x != 0 ).sum(dim=1), dtype=torch.int64)
 packed_seq = pack_padded_sequence(embeds, len_seq, batch_first=True, enforce_sorted=False)
 out_packed_lstm,(h_lstm, c_lstm)= self.lstm(packed_seq)
 if self.pooling:
out_lstm, _ = pad_packed_sequence(out_packed_lstm, batch_first=True)
 mean_out = torch.mean(out_lstm, dim=1)
 max_out, _ = torch.max(out_lstm, dim=1)
 h_lstm = torch.cat([mean_out, max_out], dim=1)
 h_lstm = h_lstm.view(h_lstm.shape[0], -1)
 if self.bidirectional:
h_lstm = self.fc_b_pool(h_lstm)
 else:
h_lstm = h_lstm.transpose(0,1)
 h_lstm = h_lstm.contiguous().view(h_lstm.shape[0], -1)
 out = self.dropout(self.fc(h_lstm))
out = self.fc_out(out)
 return out 
<choose_model_class>
data_all['tfidf_cosineSim_search_title'] = arr1
data_all['tfidf_cosineSim_search_description'] = arr2
data_all['tfidf_cosineSim_search_attribute'] = arr3
def train_model(model, train_dataloader, val_dataloader, learning_rate=1e-3, epochs=50, early_stopping=10, model_name='model_disaster_lstm.pt'):
PATH_OUTPUT_MODEL = '/kaggle/working'
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=0.01)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.5)
 criterion = nn.BCEWithLogitsLoss()
min_val_loss = None
patience = 0
list_lr, list_acc, list_loss, list_val_acc, list_val_loss = [], [], [], [], []
model = model.to(device)
 for i in tqdm(range(epochs), desc='epochs'):
n_total, n_correct, loss_total = 0, 0, 0
model.train()
for sample_batched in tqdm(train_dataloader, desc='batch loader'):
optimizer.zero_grad()
inputs = sample_batched['token_ids'].to(device)
 outputs = model(inputs)
 targets = sample_batched['target'].to(device ).float().unsqueeze(1)
 loss = criterion(outputs, targets)
 loss.backward()
optimizer.step()
with torch.no_grad() :
n_correct += get_n_correct(targets, outputs)
 n_total += len(outputs)
 loss_total += loss.item() * len(outputs)
 train_acc = n_correct / n_total
train_loss = loss_total / n_total
learning_rate = optimizer.param_groups[0]['lr']
print(f'Epoch {i} :')
 print('LR:', learning_rate)
 print('Acc:', train_acc)
 print('Loss:', train_loss)
 val_acc, val_loss = evaluate(val_dataloader, criterion, model)
 if min_val_loss == None:
min_val_loss = val_loss
if min_val_loss > val_loss:
min_val_loss = val_loss
print('Found Best Val Loss...')
 torch.save(model.state_dict() , os.path.join(PATH_OUTPUT_MODEL, model_name))
patience = 0
else:
patience += 1
scheduler.step()
print('Val Acc:', val_acc)
 print('Val Loss:', val_loss)
 list_lr.append(learning_rate)
 list_loss.append(train_loss)
 list_val_loss.append(val_loss)
 list_acc.append(train_acc)
 list_val_acc.append(val_acc)
 if patience == early_stopping:
print(f'Validation Loss not decreasing {early_stopping} times in a row.. ')
 print('Early Stopping.. ')
 print('Best Val Loss:', min_val_loss)
 break
return min_val_loss,(list_lr, list_acc, list_loss, list_val_acc, list_val_loss )<create_dataframe>
data_all['tfidf_cosineSim_search_title'] = data_all['tfidf_cosineSim_search_title'].fillna(data_all['tfidf_cosineSim_search_title'].sum() /len(data_all))
data_all['tfidf_cosineSim_search_description'] = data_all['tfidf_cosineSim_search_description'].fillna(data_all['tfidf_cosineSim_search_description'].sum() /len(data_all))
class MetricsBuilder(object):
def __init__(self):
self.df_metrics = pd.DataFrame({})
 def add_metrics(self, tuple_metrics, name='LSTM'):
list_lr, list_acc, list_loss, list_val_acc, list_val_loss = tuple_metrics
metrics_data = pd.DataFrame({'acc':list_acc, 'val_acc':list_val_acc, 'loss':list_loss, 'val_loss':list_val_loss, 'lr':list_lr,
'metrics': [name for i in range(len(list_lr)) ], 'epochs':[i for i in range(len(list_lr)) ]})
 self.df_metrics = pd.concat([self.df_metrics, metrics_data])
 def __get_df_metrics__(self):
return self.df_metrics
metrics_builder = MetricsBuilder()<train_model>
arr1 = []
arr2 = []
arr3 = []
for i in range(len(data_all)) :
product_title_i = data_all['product_title'][i]
description_i = data_all['product_description'][i]
attribute_i = data_all['attribute'][i]
search_term_i = data_all['search_term'][i]
vectorizer1 = CountVectorizer()
vectorizer2 = CountVectorizer()
vectorizer3 = CountVectorizer()
tf1 = vectorizer1.fit_transform([product_title_i, search_term_i])
 tf2 = vectorizer2.fit_transform([description_i, search_term_i])
 tf3 = vectorizer3.fit_transform([attribute_i, search_term_i])
 cosineSim_title_search = cosineSim(tf1.toarray() [0], tf1.toarray() [1])
 arr1.append(cosineSim_title_search)
 cosineSim_description = cosineSim(tf2.toarray() [0], tf2.toarray() [1])
 arr2.append(cosineSim_description)
 cosineSim_attribute = cosineSim(tf3.toarray() [0], tf3.toarray() [1])
 arr3.append(cosineSim_attribute )
lstm_model = LSTMModel(embedding_matrix, bidirectional=False)
 min_val_loss_lstm, metrics_lstm = train_model(lstm_model, train_dataloader, val_dataloader, model_name='lstm.pt' )<choose_model_class>
data_all['cosineSim_search_title'] = arr1
data_all['cosineSim_search_description'] = arr2
data_all['cosineSim_search_attribute'] = arr3
lstm_model_pool = LSTMModel(embedding_matrix, bidirectional=False, pooling=True)
 min_val_loss_lstm_pool, metrics_lstm_pool = train_model(lstm_model_pool, train_dataloader, val_dataloader, model_name='lstm_pool.pt' )<train_model>
data_all['product_info'] = data_all['search_term']+'\t'+data_all['product_title']+'\t'+data_all['product_description']+'\t'+data_all['attribute']
data_all['word_in_title'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0],str(x ).split('\t')[1]))
data_all['word_in_description'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0],str(x ).split('\t')[2]))
data_all['word_in_attributes'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0],str(x ).split('\t')[3]))
data_all['ratio_title'] = data_all['word_in_title']/data_all['number_of_words_in_search']
data_all['ratio_description'] = data_all['word_in_description']/data_all['number_of_words_in_search']
data_all['ratio_attributes'] = data_all['word_in_attributes']/data_all['number_of_words_in_search']
data_all['title_length'] = data_all.product_title.str.len()
bilstm_model = LSTMModel(embedding_matrix, bidirectional=True)
 min_val_loss_bilstm, metrics_bilstm = train_model(bilstm_model, train_dataloader, val_dataloader, model_name='bilstm.pt' )<load_pretrained>
data_all['attr'] = str(data_all['search_term'])+"\t"+str(data_all['brand'])
 data_all['brand_in_search'] = data_all['attr'].map(lambda x:str_common_word(str(x ).split('\t')[0],str(x ).split('\t')[1]))
data_all['ratio_brand'] = data_all['brand_in_search']/data_all['len_of_brand']
bilstm_model_pool = LSTMModel(embedding_matrix, bidirectional=True, pooling=True)
 min_val_loss_bilstm_pool, metrics_bilstm_pool = train_model(bilstm_model_pool, train_dataloader, val_dataloader, model_name='bilstm_pool.pt' )<train_model>
data_all['ratio_title'] = data_all['ratio_title'].fillna(data_all['ratio_title'].sum() /len(data_all))
data_all['ratio_description'] = data_all['ratio_description'].fillna(data_all['ratio_description'].sum() /len(data_all))
data_all['ratio_attributes'] = data_all['ratio_attributes'].fillna(data_all['ratio_attributes'].sum() /len(data_all))
print('min_val_loss_bilstm',min_val_loss_bilstm)
 print('min_val_loss_bilstm_pool',min_val_loss_bilstm_pool)
 print('min_val_loss_lstm',min_val_loss_lstm)
 print('min_val_loss_lstm_pool',min_val_loss_lstm_pool )<compute_train_metric>
def fuzzy_partial_ratio(string_1 , string_2):
return fuzz.partial_ratio(string_1, string_2 )
metrics_builder.add_metrics(metrics_lstm, name='LSTM')
 metrics_builder.add_metrics(metrics_lstm_pool, name='LSTM_Pool')
 metrics_builder.add_metrics(metrics_bilstm, name='BiLSTM')
 metrics_builder.add_metrics(metrics_bilstm_pool, name='BiSTM_Pool' )<find_best_params>
def fuzzy_token_sort_ratio(string_1,string_2):
return fuzz.token_sort_ratio(string_1,string_2 )
df_metrics = metrics_builder.__get_df_metrics__()<load_pretrained>
data_all['fuzzy_ratio_in_title'] = data_all['product_info'].map(lambda x:fuzzy_partial_ratio(x.split('\t')[0],x.split('\t')[1]))
data_all['fuzzy_ratio_in_description'] = data_all['product_info'].map(lambda x:fuzzy_partial_ratio(x.split('\t')[0],x.split('\t')[2]))
data_all['fuzzy_ratio_in_attribute'] = data_all['product_info'].map(lambda x:fuzzy_partial_ratio(x.split('\t')[0],x.split('\t')[3]))
best_bilstm_pool = LSTMModel(embedding_matrix, bidirectional=True, pooling=True ).to(device)
 best_bilstm_pool.load_state_dict(torch.load('/kaggle/working/bilstm_pool.pt', map_location=device))<predict_on_test>
data_all['fuzzy_token_sort_ratio_in_title'] = data_all['product_info'].map(lambda x:fuzzy_token_sort_ratio(x.split('\t')[0],x.split('\t')[1]))
data_all['fuzzy_token_sort_ratio_in_description'] = data_all['product_info'].map(lambda x:fuzzy_token_sort_ratio(x.split('\t')[0],x.split('\t')[2]))
data_all['fuzzy_token_sort_ratio_in_attribute'] = data_all['product_info'].map(lambda x:fuzzy_token_sort_ratio(x.split('\t')[0],x.split('\t')[3]))
def predict_test_set(data_loader, model):
model.eval()
all_outputs = []
with torch.no_grad() :
for sample_batched in data_loader:
inputs = sample_batched.to(device)
 outputs = model(inputs)
 outputs =(torch.sigmoid(outputs ).squeeze(1)> 0.5 ).long().tolist()
all_outputs.extend(outputs)
 return all_outputs 
<string_transform>
tf = TfidfVectorizer()
arr1 =[]
arr2 = []
arr3 = []
arr4 = []
for i in range(len(data_all)) :
arr1.append(tf.fit_transform([data_all['product_description'][i]] ).mean())
 arr3.append(tf.fit_transform([data_all['product_title'][i]] ).mean())
 arr4.append(tf.fit_transform([data_all['attribute'][i]] ).mean())
 data_all['mean_product_des'] = arr1
data_all['mean_product_title'] = arr3
data_all['mean_attribute'] = arr4
def generate_test_data_loader(x, maxlen=128, bs=16):
all_data = []
for text in x:
tokens = word_tokenize(text)
 token_ids = [word2idx[t] for t in tokens]
if len(token_ids)> maxlen:
token_ids = token_ids[:maxlen]
else:
token_ids = token_ids +([0] *(maxlen - len(token_ids)))
 token_ids_tensor = torch.LongTensor(token_ids)
 all_data.append(token_ids_tensor)
 data_loader = DataLoader(all_data, batch_size=bs, num_workers=5)
 return data_loader<predict_on_test>
data_all['last_word_title_match'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0].split(" ")[-1],str(x ).split('\t')[1]))
test_dataloader = generate_test_data_loader(preprocess(test['text']))
predicted_target = predict_test_set(test_dataloader, best_bilstm_pool)
<save_to_csv>
data_all['last_word_description_match'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0].split(" ")[-1],str(x ).split('\t')[2]))
my_submission = pd.DataFrame({'Id': test.id, 'target': predicted_target})
 my_submission.to_csv('submission.csv', index=False)
<set_options>
data_all['first_word_title_match'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0].split(" ")[0],str(x ).split('\t')[1]))
%matplotlib inline<import_modules>
data_all['first_word_description_match'] = data_all['product_info'].map(lambda x:str_common_word(str(x ).split('\t')[0].split(" ")[0],x.split('\t')[2]))
from keras.models import Sequential
from keras.layers import Dense , Dropout , Lambda, Flatten
from keras.optimizers import Adam ,RMSprop
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras.preprocessing.image import ImageDataGenerator<load_from_csv>
data_all_test = data_all
train = pd.read_csv(".. /input/train.csv")
 print(train.shape )<load_from_csv>
data_all = data_all.drop(['product_title', 'search_term', 'product_description', 'search_term_tokens', 'product_title_tokens', 'product_description_tokens', 'product_info', 'attr', 'attribute', 'brand'], axis = 1 )
test= pd.read_csv(".. /input/test.csv")
 print(test.shape )<data_type_conversions>
train = data_all[:74067]
test = data_all[74067:]
X_train =(train.iloc[:,1:].values ).astype('float32')
 y_train = train.iloc[:,0].values.astype('int32')
 X_test = test.values.astype('float32' )<data_type_conversions>
y_train = train['relevance'].values
X_train = train.drop(['id', 'relevance'], axis = 1 ).values
mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px<categorify>
id_test = test['id']
X_test = test.drop(['id', 'relevance'], axis = 1 ).values
y_train= to_categorical(y_train)
 num_classes = y_train.shape[1]
num_classes<define_variables>
xgb = XGBRegressor()
param_grid = {'max_depth':[5, 6],
'n_estimators': [130, 150, 170],
'learning_rate' : [0.1]}
model_xgb = sklearn.model_selection.GridSearchCV(estimator = xgb, param_grid = param_grid, n_jobs = -1)
 model_xgb.fit(X_train, y_train)
 y_pred = model_xgb.predict(X_test )
seed = 10
np.random.seed(seed )<import_modules>
for i in range(len(y_pred)) :
if y_pred[i] > 3:
y_pred[i] = 3
from keras.models import Sequential
from keras.layers.core import Lambda , Dense, Flatten, Dropout
from keras.callbacks import EarlyStopping
from keras.layers import BatchNormalization, Convolution2D , MaxPooling2D<train_model>
pd.DataFrame({"id": id_test, "relevance": y_pred} ).to_csv('submission.csv',index=False )
model= Sequential()
model.add(Lambda(standardize,input_shape=(28,28,1)))
 model.add(Flatten())
 model.add(Dense(10, activation='softmax'))
print("input shape ",model.input_shape)
 print("output shape ",model.output_shape )<choose_model_class>
def caculator_RMSE(X_train, y_train, model):
return np.sqrt(((y_train - model.predict(X_train)) ** 2 ).mean() )
model.compile(optimizer=RMSprop(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'] )<split>
print('RMSE using XGBRegression : ' , caculator_RMSE(X_train, y_train, model_xgb))
X = X_train
y = y_train
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=10)
 batches = gen.flow(X_train, y_train, batch_size=64)
 val_batches=gen.flow(X_val, y_val, batch_size=64 )<train_model>
test1 = data_all_test[:74067]
test2 = data_all_test[74067:]
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3,
validation_data=val_batches, validation_steps=val_batches.n )<choose_model_class>
test2.isnull().sum()
def get_cnn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28,28,1)) ,
Convolution2D(32,(3,3), activation='relu'),
Convolution2D(32,(3,3), activation='relu'),
MaxPooling2D() ,
Convolution2D(64,(3,3), activation='relu'),
Convolution2D(64,(3,3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
Dense(512, activation='relu'),
Dense(10, activation='softmax')
 ])
 model.compile(Adam() , loss='categorical_crossentropy',
metrics=['accuracy'])
 return model<choose_model_class>
X = test1.drop(['id','relevance','product_title', 'search_term', 'product_description', 'search_term_tokens', 'product_title_tokens', 'product_description_tokens', 'product_info', 'attr', 'attribute', 'brand'], axis = 1)
model= get_cnn_model()
model.optimizer.lr=0.01<train_model>
y = test1['relevance']
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1,
validation_data=val_batches, validation_steps=val_batches.n )<choose_model_class>
id_test = test2['id']
X_test = test2.drop(['id','relevance','product_title', 'search_term', 'product_description', 'search_term_tokens', 'product_title_tokens', 'product_description_tokens', 'product_info', 'attr', 'attribute', 'brand'], axis = 1 )
def get_bn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28,28,1)) ,
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(32,(3,3), activation='relu'),
MaxPooling2D() ,
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
BatchNormalization() ,
Dense(512, activation='relu'),
BatchNormalization() ,
Dense(10, activation='softmax')
 ])
 model.compile(Adam() , loss='categorical_crossentropy', metrics=['accuracy'])
 return model<train_model>
model = LinearRegression().fit(X, y)
 y_pred = model.predict(X_test)
 print("RMSE using LinearRegressor:", caculator_RMSE(X, y, model))
model= get_bn_model()
model.optimizer.lr=0.01
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1,
validation_data=val_batches, validation_steps=val_batches.n )<train_model>
regressor = RandomForestRegressor(n_estimators=20, random_state=0)
 regressor.fit(X, y)
 y_pred = regressor.predict(X_test)
 print("RMSE using RandomForest:", caculator_RMSE(X, y, regressor))
model.optimizer.lr=0.01
gen = image.ImageDataGenerator()
batches = gen.flow(X, y, batch_size=64)
 history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3 )<save_to_csv>
param_grid = {
'loss' : ['ls'],
'n_estimators' : [3],
'max_depth' : [9],
'max_features' : ['auto']
}
gbr = GradientBoostingRegressor()
model_gbr = sklearn.model_selection.GridSearchCV(estimator = gbr, n_jobs = -1, param_grid = param_grid)
 model_gbr.fit(X, y)
 y_pred = model_gbr.predict(X_test)
 print("RMSE using GBR:", caculator_RMSE(X, y, model_gbr))
<set_options><EOS> 
gnb = BayesianRidge()
param_grid = {}
model_nb = sklearn.model_selection.GridSearchCV(estimator = gnb, param_grid = param_grid, n_jobs = -1)
 model_nb.fit(X, y)
 y_pred = model_nb.predict(X_test)
 print("RMSE using Nayve bayes:", caculator_RMSE(X, y, model))