Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<choose_model_class>
%matplotlib inline
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='Mish' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='Mish' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
data_train = pd.read_csv('/kaggle/input/mercari-price-suggestion-challenge/train.tsv', delimiter='\t' )
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )<save_to_csv>
data_test = pd.read_csv('/kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv', delimiter='\t' )
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<import_modules>
print("Train DataFrame")
 print(pd.isnull(data_train ).sum())
 print("="*50)
 print("Test DataFrame")
 print(pd.isnull(data_test ).sum() )
print(os.listdir(".. /input")) 
<load_from_csv>
print(( pd.isnull(data_train['brand_name'] ).sum())*100/data_train.shape[0])
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train= pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv' )<feature_engineering>
print("{0:.2f}% brand_name has missing value in train data".format(( pd.isnull(data_train['brand_name'] ).sum())*100/data_train.shape[0]))
print("{0:.2f}% category_name has missing value in train data".format(( pd.isnull(data_train['category_name'] ).sum())*100/data_train.shape[0]))
emoji_dict = defaultdict()
with open('.. /input/emoji-unicode-names/emoji_unicode_names_final.txt', 'r', encoding="utf8")as f:
lines = f.readlines()
for line in lines:
tokens = line.strip().split('\t')
 emoji_dict[tokens[0]] = tokens[1]<define_variables>
print("{0:.2f}% brand_name has missing value in test data".format(( pd.isnull(data_test['brand_name'] ).sum())*100/data_test.shape[0]))
print("{0:.2f}% category_name has missing value in test data".format(( pd.isnull(data_test['category_name'] ).sum())*100/data_test.shape[0]))
repl = {
"&lt;3": " good ",
":d": " good ",
":dd": " good ",
":p": " good ",
"8)": " good ",
":-)": " good ",
":)": " good ",
";)": " good ",
"(-:": " good ",
"(:": " good ",
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " bad ",
":(": " bad ",
":s": " bad ",
":-s": " bad ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]<string_transform>
data_train['brand_name'] = data_train['brand_name'].fillna('Unknown')
 data_test['brand_name'] = data_test['brand_name'].fillna('Unknown' )
new_train_data = []
new_test_data = []
c_train = 0
c_test = 0
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
c_train += 1
j = repl[j]
if j in emoji_dict:
c_train += 1
j = emoji_dict[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
c_test += 1
j = repl[j]
if j in emoji_dict:
c_test += 1
j = emoji_dict[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
print("replacements in train data : ", c_train)
 print("replacements in test data : ", c_test )<drop_column>
data_train['category_name'] = data_train['category_name'].fillna('No/No/No')
 data_test['category_name'] = data_test['category_name'].fillna('No/No/No' )
train = train.drop("comment_text", axis=1)
 test = test.drop("comment_text", axis=1 )<prepare_x_and_y>
def find_sub_cat(X):
try:
return(X.split("/"))
return("jubu")
 except:
return("None","None","None")
train["new_comment_text"].fillna("fillna")
 test["new_comment_text"].fillna("fillna")
 X_train = train["new_comment_text"]
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["new_comment_text"]<define_variables>
data_train['main_category']=''
data_train['sub_category_1']=''
data_train['sub_category_2']=''
data_train['main_category'],data_train['sub_category_1'],data_train['sub_category_2']\
= zip(*data_train['category_name'].apply(lambda x: find_sub_cat(x)))
max_features=100000
maxlen=200
embed_size=300<compute_train_metric>
data_test['main_category']=''
data_test['sub_category_1']=''
data_test['sub_category_2']=''
data_test['main_category'],data_test['sub_category_1'],data_test['sub_category_2']\
= zip(*data_test['category_name'].apply(lambda x: find_sub_cat(x)))
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<categorify>
brands = data_train['brand_name'].value_counts()
print(brands[:10] )
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )<categorify>
( data_train['shipping'].value_counts())*100/data_train.shape[0]
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs 
<feature_engineering>
print("There are %d unique main_categories." % data_train['main_category'].nunique() )
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<init_hyperparams>
print("There are %d unique sub_category_1." % data_train['sub_category_1'].nunique() )
gru_len = 128
Routings = 5
Num_capsule = 10
Dim_capsule = 16
dropout_p = 0.25
rate_drop_dense = 0.28
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 def get_model() :
input1 = Input(shape=(maxlen,))
embed_layer = Embedding(max_features,
embed_size,
input_length=maxlen,
weights=[embedding_matrix],
trainable=False )(input1)
 embed_layer = SpatialDropout1D(rate_drop_dense )(embed_layer)
 x = Bidirectional(
 GRU(gru_len, activation='relu', dropout=dropout_p, recurrent_dropout=dropout_p, return_sequences=True))(
 embed_layer)
 capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,
share_weights=True )(x)
 capsule = Flatten()(capsule)
 capsule = Dropout(dropout_p )(capsule)
 output = Dense(6, activation='sigmoid' )(capsule)
 model = Model(inputs=input1, outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.summary()
return model<split>
print("There are %d unique sub_category_2." % data_train['sub_category_2'].nunique() )
model = get_model()
batch_size = 256
epochs = 3
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95, random_state=233)
 RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1 )<train_model>
stopwords= ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've",\
"you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \
'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their',\
'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', \
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \
'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \
'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\
'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\
'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\
'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \
's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', \
've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn',\
"hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn',\
"mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", \
'won', "won't", 'wouldn', "wouldn't"]
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=1 )<train_model>
def decontracted(phrase):
try:
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
except:
return 0
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=1 )<train_model>
def wordCount_with_cleaning(sentance):
try:
sent = decontracted(sentance)
 sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 sent = sent.strip()
return(len(sent.split()))
except:
return 0
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=1 )<predict_on_test>
def wordCount_without_cleaning(sentance):
try:
sent = ' '.join(e for e in sentance.split())
 sent = sent.strip()
return(len(sent.split()))
except:
return 0
y_pred = model.predict(x_test, batch_size=1024, verbose=1)
<save_to_csv>
preprocessed_item_description = []
for sentance in tqdm(data_train['item_description'].values):
sent = decontracted(str(sentance))
sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 preprocessed_item_description.append(sent.lower().strip() )
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False)
 model.save_weights('best.hdf5')
<import_modules>
preprocessed_test_item_description = []
for sentance in tqdm(data_test['item_description'].values):
sent = decontracted(str(sentance))
sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 preprocessed_test_item_description.append(sent.lower().strip() )
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, GRU, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from nltk.tokenize import TweetTokenizer
from unidecode import unidecode
import gc
from keras import backend as K<import_modules>
data_train = data_train.drop('item_description',axis=1)
 data_train['item_description'] = preprocessed_item_description
data_train.head(2 )
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<define_variables>
data_test = data_test.drop('item_description',axis=1)
 data_test['item_description'] = preprocessed_test_item_description
data_test.head(2 )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE1=f'{path}glove840b300dtxt/glove.840B.300d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
EMBEDDING_FILE2=f'{path}fasttext-crawl-300d-2m/rawl-300d-2M.vec'
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]<load_from_csv>
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )<define_variables>
y = np.log10(np.array(data_train['price'])+1)
 X = data_train.drop('price',axis=1)
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<define_variables>
X_train,X_cv,Y_train,Y_cv = train_test_split(X, y, test_size=0.20, random_state=42)
embedding_dim_facebook = 300
embedding_dim_google = 300
max_features = 20000
maxlen = 100<define_variables>
def concat_categories(x):
return set(x.values)
 def brandfinder(name, category):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
return brand
return 'Unknown'
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {} 
<define_variables>
brand_names_categories = dict(X_train[X_train['brand_name'] != 'Unknown'][['brand_name','category_name']].astype('str')\ 
.groupby('brand_name' ).agg(concat_categories ).reset_index().values.tolist())\
brands_sorted_by_size = list(sorted(filter(lambda y: len(y)>= 3, \
list(brand_names_categories.keys())) , \
key = lambda x: -len(x)) )
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
train_names_unknown_brands_train = X_train[X_train['brand_name'] == 'Unknown'][['name','category_name']].\
astype('str' ).values
train_names_unknown_brands_cv = X_cv[X_cv['brand_name'] == 'Unknown'][['name','category_name']].\
astype('str' ).values
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 subject = normalized_comment
subject = re.sub(r"([a-z])\-([a-z])", r"\1 \2", subject , 0, re.IGNORECASE)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result<load_from_csv>
train_names_unknown_brands_data_test = data_test[data_test['brand_name'] == 'Unknown'][['name','category_name']].\
astype('str' ).values
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
train_estimated_brands_train = []
for name, category in tqdm(train_names_unknown_brands_train):
brand = brandfinder(name, category)
 train_estimated_brands_train.append(brand )
%%time
labels, x_train, x_test = read_data_files(TRAIN_DATA_FILE, TEST_DATA_FILE )<string_transform>
train_estimated_brands_cv = []
for name, category in tqdm(train_names_unknown_brands_cv):
brand = brandfinder(name, category)
 train_estimated_brands_cv.append(brand )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<load_pretrained>
train_estimated_brands_data_test = []
for name, category in tqdm(train_names_unknown_brands_data_test):
brand = brandfinder(name, category)
 train_estimated_brands_data_test.append(brand )
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )<split>
X_train.loc[X_train['brand_name'] == 'Unknown', 'brand_name'] = train_estimated_brands_train
X_cv.loc[X_cv['brand_name'] == 'Unknown', 'brand_name'] = train_estimated_brands_cv
data_test.loc[data_test['brand_name'] == 'Unknown', 'brand_name'] = train_estimated_brands_data_test
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1 )<choose_model_class>
max_brand = 2500
max_category_name = 1000
max_category = 1000
name_min_df =10
max_item_description_features = 50000
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
keep_brand = X_train['brand_name'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_brand]
X_train.loc[~X_train['brand_name'].isin(keep_brand), 'brand_name'] = 'Unknown'
X_cv.loc[~X_cv['brand_name'].isin(keep_brand), 'brand_name'] = 'Unknown'
data_test.loc[~data_test['brand_name'].isin(keep_brand), 'brand_name'] = 'Unknown'
%%time
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )<save_to_csv>
keep_category1 = X_train['main_category'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_category]
keep_category2 = X_train['sub_category_1'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_category]
keep_category3 = X_train['sub_category_2'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_category]
X_train.loc[~X_train['main_category'].isin(keep_category1), 'main_category'] = 'Unknown'
X_train.loc[~X_train['sub_category_1'].isin(keep_category2), 'sub_category_1'] = 'Unknown'
X_train.loc[~X_train['sub_category_2'].isin(keep_category3), 'sub_category_1'] = 'Unknown'
X_cv.loc[~X_cv['main_category'].isin(keep_category1), 'main_category'] = 'Unknown'
X_cv.loc[~X_cv['sub_category_1'].isin(keep_category2), 'sub_category_1'] = 'Unknown'
X_cv.loc[~X_cv['sub_category_2'].isin(keep_category3), 'sub_category_1'] = 'Unknown'
data_test.loc[~data_test['main_category'].isin(keep_category1), 'main_category'] = 'Unknown'
data_test.loc[~data_test['sub_category_1'].isin(keep_category2), 'sub_category_1'] = 'Unknown'
data_test.loc[~data_test['sub_category_2'].isin(keep_category3), 'sub_category_1'] = 'Unknown'
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<define_variables>
tv = TfidfVectorizer(max_features=max_item_description_features,ngram_range=(1, 3),token_pattern=r'(?u)\b\w+\b',stop_words='english')
 tv.fit(X_train['item_description'])
 X_train_item_description_tfidf = tv.transform(X_train['item_description'])
 X_cv_item_description_tfidf = tv.transform(X_cv['item_description'])
 data_test_item_description_tfidf = tv.transform(data_test['item_description'] )
f_nbsvm = '.. /input/nb-svm-strong-linear-baseline/submission.csv'
f_bi_lstm_dual = '.. /input/bi-gru-lstm-dual-embedding-with-mish/submission.csv'<load_from_csv>
tv.fit(X_train['name'])
 X_train_name_tfidf = tv.transform(X_train['name'])
 X_cv_name_tfidf = tv.transform(X_cv['name'])
 data_test_name_tfidf = tv.transform(data_test['name'] )
p_nbsvm = pd.read_csv(f_nbsvm)
 p_bi_lstm_dual = pd.read_csv(f_bi_lstm_dual )<feature_engineering>
cv = CountVectorizer(min_df=name_min_df)
 cv.fit(X_train['category_name'].astype('category'))
X_train_category =cv.transform(X_train['category_name'].astype('category'))
X_cv_category =cv.transform(X_cv['category_name'].astype('category'))
data_test_category =cv.transform(data_test['category_name'].astype('category'))
label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
p_res = p_nbsvm.copy()
p_res[label_cols] =(p_nbsvm[label_cols] + p_bi_lstm_dual[label_cols])/ 2<save_to_csv>
lb = LabelBinarizer(sparse_output=True)
 lb.fit(X_train['brand_name'].astype('category'))
X_train_brand = lb.transform(X_train['brand_name'].astype('category'))
X_cv_brand = lb.transform(X_cv['brand_name'])
 data_test_brand = lb.transform(data_test['brand_name'] )
p_res.to_csv('submission.csv', index=False )<import_modules>
X_train_dummies = csr_matrix(pd.get_dummies(X_train[['item_condition_id', 'shipping']], sparse=True ).values)
 X_cv_dummies = csr_matrix(pd.get_dummies(X_cv[['item_condition_id', 'shipping']], sparse=True ).values)
 data_test_dummies = csr_matrix(pd.get_dummies(data_test[['item_condition_id', 'shipping']], sparse=True ).values )
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, GRU, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from nltk.tokenize import TweetTokenizer
from unidecode import unidecode
import gc
from keras import backend as K<import_modules>
X_train_word_count_item_desc = X_train['item_description'].apply(lambda x: wordCount_with_cleaning(x))
X_cv_word_count_item_desc = X_cv['item_description'].apply(lambda x: wordCount_with_cleaning(x))
data_test_word_count_item_desc = data_test['item_description'].apply(lambda x: wordCount_with_cleaning(x))
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<define_variables>
X_train_word_count_name = X_train['name'].apply(lambda x: wordCount_with_cleaning(x))
X_cv_word_count_name = X_cv['name'].apply(lambda x: wordCount_with_cleaning(x))
data_test_word_count_name = data_test['name'].apply(lambda x: wordCount_with_cleaning(x))
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE1=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
EMBEDDING_FILE2=f'{path}fasttext-crawl-300d-2m/rawl-300d-2M.vec'
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]<load_from_csv>
X_train_vectorized = hstack(( X_train_item_description_tfidf,\
X_train_name_tfidf,\
X_train_category,\
X_train_brand,\
X_train_dummies,\
X_train_word_count_item_desc.values.reshape(-1,1),\
X_train_word_count_name.values.reshape(-1,1)) ).tocsr()
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )<define_variables>
X_cv_vectorized = hstack(( X_cv_item_description_tfidf,\
X_cv_name_tfidf,\
X_cv_category,\
X_cv_brand,\
X_cv_dummies,\
X_cv_word_count_item_desc.values.reshape(-1,1),\
X_cv_word_count_name.values.reshape(-1,1)) ).tocsr()
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<define_variables>
data_test_vectorized = hstack(( data_test_item_description_tfidf,\
data_test_name_tfidf,\
data_test_category,\
data_test_brand,\
data_test_dummies,\
data_test_word_count_item_desc.values.reshape(-1,1),\
data_test_word_count_name.values.reshape(-1,1)) ).tocsr()
embedding_dim_facebook = 300
embedding_dim_google = 300
max_features = 20000
maxlen = 100<define_variables>
del(data_train,preprocessed_item_description,preprocessed_test_item_description,X,y)
 del(train_estimated_brands_train,train_estimated_brands_cv,train_estimated_brands_data_test,keep_brand)
 del(data_test_item_description_tfidf,
data_test_name_tfidf,\
data_test_category,\
data_test_brand,\
data_test_dummies,\
data_test_word_count_item_desc,\
data_test_word_count_name)
 del(X_train_item_description_tfidf,
X_train_name_tfidf,\
X_train_category,\
X_train_brand,\
X_train_dummies,\
X_train_word_count_item_desc,\
X_train_word_count_name)
 del(X_cv_item_description_tfidf,
X_cv_name_tfidf,\
X_cv_category,\
X_cv_brand,\
X_cv_dummies,\
X_cv_word_count_item_desc,\
X_cv_word_count_name)
 gc.collect()
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}<define_variables>
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)) )
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
Error_dict = {}
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result<load_from_csv>
params = {
'colsample_bytree': 0.42799939792816927,
'max_depth': 10,
'min_child_samples': 370,
'min_child_weight': 0.01,
'num_leaves': 49,
'reg_lambda': 5,
'subsample': 0.6739316550896339,
'learning_rate':0.1,
'reg_alpha' :0.5,
'boosting_type': 'gbdt',
'objective' : 'regression',
'metric' : 'RMSE',
'verbosity': -1
}
d_train = lgb.Dataset(X_train_vectorized, label=Y_train)
 d_valid = lgb.Dataset(X_cv_vectorized, label=Y_cv)
 watchlist = [d_train, d_valid]
model_1 = lgb.train(params, train_set=d_train,valid_sets=watchlist,num_boost_round=2000,verbose_eval=200,early_stopping_rounds=100)
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
lgbm_1_pred_train=model_1.predict(X_train_vectorized)
 lgbm_1_pred_cv=model_1.predict(X_cv_vectorized)
 lgbm_1_pred_data_test=model_1.predict(data_test_vectorized )
%%time
labels, x_train, x_test = read_data_files(TRAIN_DATA_FILE, TEST_DATA_FILE )<prepare_x_and_y>
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** lgbm_1_pred_train-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** lgbm_1_pred_cv-1)) )
<string_transform>
Error_dict['lgbm_1_Train_rmsle'] = rmsle(10 ** Y_train-1, 10 ** lgbm_1_pred_train-1)
 Error_dict['lgbm_1_cv_rmsle'] = rmsle(10 ** Y_cv-1, 10 ** lgbm_1_pred_cv-1 )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<load_pretrained>
reg_ridge = Ridge(solver='sag', alpha=5)
 reg_ridge.fit(X_train_vectorized, Y_train )
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )<split>
reg_ridge_pred_train=reg_ridge.predict(X_train_vectorized)
 reg_ridge_pred_cv=reg_ridge.predict(X_cv_vectorized)
 reg_ridge_pred_data_test=reg_ridge.predict(data_test_vectorized )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1 )<choose_model_class>
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** reg_ridge_pred_train-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** reg_ridge_pred_cv-1)) )
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
Error_dict['reg_ridge_Train_rmsle'] = rmsle(10 ** Y_train-1, 10 ** reg_ridge_pred_train-1)
 Error_dict['reg_ridge_Cv_rmsle'] = rmsle(10 ** Y_cv-1, 10 ** reg_ridge_pred_cv-1 )
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )<string_transform>
stacked_pred_train = pd.DataFrame(index=np.array(range(X_train.shape[0])))
 stacked_pred_cv = pd.DataFrame(index=np.array(range(X_cv.shape[0])))
 stacked_pred_data_test = pd.DataFrame(index=np.array(range(data_test.shape[0])))
 stacked_pred_train.insert(0,"lgbm_model_pred",lgbm_1_pred_train)
 stacked_pred_train.insert(1,"Ridge_model_pred",reg_ridge_pred_train)
 stacked_pred_cv.insert(0,"lgbm_model_pred",lgbm_1_pred_cv)
 stacked_pred_cv.insert(1,"Ridge_model_pred",reg_ridge_pred_cv)
 stacked_pred_data_test.insert(0,"lgbm_model_pred",lgbm_1_pred_data_test)
 stacked_pred_data_test.insert(1,"Ridge_model_pred",reg_ridge_pred_data_test)
<choose_model_class>
X_train_stacked = sparse.csr_matrix(hstack([X_train_vectorized, sparse.csr_matrix(stacked_pred_train)]))
X_cv_stacked = sparse.csr_matrix(hstack([X_cv_vectorized, sparse.csr_matrix(stacked_pred_cv)]))
data_test_stacked = sparse.csr_matrix(hstack([data_test_vectorized, sparse.csr_matrix(stacked_pred_data_test)]))
<save_to_csv>
params = {
'colsample_bytree': 0.42799939792816927,
'max_depth': 10,
'min_child_samples': 370,
'min_child_weight': 0.01,
'num_leaves': 49,
'reg_lambda': 5,
'subsample': 0.6739316550896339,
'learning_rate':0.1,
'reg_alpha' :0.5,
'boosting_type': 'gbdt',
'objective' : 'regression',
'metric' : 'RMSE',
'verbosity': -1
}
d_train = lgb.Dataset(X_train_stacked, label=Y_train)
 d_valid = lgb.Dataset(X_cv_stacked, label=Y_cv)
 watchlist = [d_train, d_valid]
final_lgbm = lgb.train(params, train_set=d_train,valid_sets=watchlist,num_boost_round=2000,verbose_eval=200,early_stopping_rounds=100 )
<save_to_csv>
final_pred_train=final_lgbm.predict(X_train_stacked)
 final_pred_cv=final_lgbm.predict(X_cv_stacked)
 final_pred_data_test=final_lgbm.predict(data_test_stacked )
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<import_modules>
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** final_pred_train-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** final_pred_cv-1)) )
<load_from_csv><EOS> 
submission = pd.DataFrame()
submission['test_id'] = data_test['test_id']
lightgbm_submission = submission.copy()
lightgbm_submission['price'] = pd.DataFrame(10 ** final_pred_data_test - 1)
 lightgbm_submission.to_csv('stacked_submission_1.csv', index=False )