Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<define_variables>
%matplotlib inline
seed = 90
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
data_folder = ".. /input/jigsaw-toxic-comment-classification-challenge/"
pretrained_folder = ".. /input/"
train_filepath = data_folder + "train.csv"
test_filepath = data_folder + "test.csv"
submission_path = data_folder + "submission.csv"
facebook_filepath = pretrained_folder + "fasttext-crawl-300d-2m/crawl-300d-2M.vec"
google_filepath = pretrained_folder + "glove840b300dtxt/glove.840B.300d.txt"
embedding_dim_facebook = 300
embedding_dim_google = 300<load_pretrained>
Time_0 = time.time()
train = pd.read_csv('.. /input/train.tsv',sep='\t')
 test = pd.read_csv('.. /input/test.tsv',sep='\t' )
hyphens_filepath = ".. /input/cleaning-dictionaries/hyphens_dictionary.bin"
misspellings_filepath = ".. /input/cleaning-dictionaries/misspellings_all_dictionary.bin"
merged_filepath = ".. /input/cleaning-dictionaries/merged_all_dictionary.bin"
hyphens_dict = misspellings_dict = merged_dict = {}
with open(hyphens_filepath, mode='rb')as file: hyphens_dict = pickle.load(file)
 with open(misspellings_filepath, mode='rb')as file: misspellings_dict = pickle.load(file)
 with open(merged_filepath, mode='rb')as file: merged_dict = pickle.load(file)
 print(len(hyphens_dict))
print(len(misspellings_dict))
print(len(merged_dict))<define_variables>
y_train = train['log_price'] = np.log(( train['price'] + 1))
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}<define_variables>
def handle_missing(dataset):
dataset['category_name'].fillna(value="NA/NA/NA", inplace=True)
 dataset['brand_name'].fillna(value="missing", inplace=True)
 dataset['item_description'].fillna(value="missing", inplace=True)
 return(dataset)
 def split_cat(dataset):
dataset['cat1'], dataset['cat2'], dataset['cat3'] = zip(*dataset['category_name'].str.split("/",2))
return dataset
def label_maker(dataset):
lb = LabelBinarizer(sparse_output=True)
 cat1 = lb.fit_transform(dataset['cat1'])
 cat2 = lb.fit_transform(dataset['cat2'])
 cat3 = lb.fit_transform(dataset['cat3'])
 brand_name = lb.fit_transform(dataset['brand_name'])
 del lb
return cat1,cat2,cat3,brand_name
def get_dums(dataset):
X_dummies = csr_matrix(pd.get_dummies(dataset[['item_condition_id', 'shipping']],
sparse=True ).values)
 return X_dummies
def text_processing(dataset):
MIN_DF_COUNT = 10
MAX_DF_COUNT = 10000
cv = CountVectorizer(min_df = MIN_DF_COUNT, max_df = MAX_DF_COUNT)
 name = cv.fit_transform(dataset['name'])
 MIN_DF_TF = 10
MAX_DF_TF = 51000
MAX_FEATURES_TF = 51000
tv = TfidfVectorizer(max_features=MAX_FEATURES_TF,
min_df = MIN_DF_TF,
max_df = MAX_DF_TF,
ngram_range=(1, 3),
stop_words='english')
 description = tv.fit_transform(dataset['item_description'])
 del cv, tv
return name, description
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
nrow_train = train.shape[0]
merge: pd.DataFrame = pd.concat([train, test])
 submission: pd.DataFrame = test[['test_id']]
del train
del test
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, hyphens_dict)
 normalized_word = normalize_by_dictionary(normalized_word, merged_dict)
 normalized_word = normalize_by_dictionary(normalized_word, misspellings_dict)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result 
<load_from_csv>
start_time = time.time()
print("Handle Missing...")
 merge = handle_missing(merge)
 print("splitting cat...")
 merge = split_cat(merge)
 print("making labels...")
 cat1,cat2,cat3,brand_name = label_maker(merge)
 print("getting dummies...")
 X_dummies = get_dums(merge)
 print("processing text...")
 name,description = text_processing(merge)
 print("stacking train...")
 sparse_merge = hstack(( cat1,cat3,cat3,brand_name,X_dummies,name,description)).tocsr()
print("TIME:", time.time() - start_time )
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
X_train = sparse_merge[:nrow_train]
X_test = sparse_merge[nrow_train:]
%%time
labels, x_train, x_test = read_data_files(train_filepath, test_filepath )<string_transform>
def model_testing(model,X_test, y_test):
y_pred = model.predict(X_test)
 error = rmsle(y_test, y_pred)
 print(error)
 def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)))
%%time
fileObject = open('.. /input/cleaning-dictionaries/tokenizer','rb')
 tokenizer = pickle.load(fileObject)
 x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<load_pretrained>
ridge_model_1 = Ridge(alpha=5.0, fit_intercept=True, normalize=False, copy_X=True,
max_iter=None, tol=0.001, solver='auto', random_state=None)
 ridge_model_2 = Ridge(alpha=5.0, fit_intercept=True, normalize=False, copy_X=True,
max_iter=None, tol=0.001, solver='sag', random_state=None)
 ridge_model_3 = Ridge(alpha=5.0, fit_intercept=True, normalize=False, copy_X=True,
max_iter=None, tol=0.001, solver='lsqr', random_state=None)
 gbrt = GradientBoostingRegressor(max_depth = 2, n_estimators = 5,
learning_rate = 0.9,subsample=0.9 )
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )<split>
start_time = time.time()
print("train test splitting...")
 X_t, X_v, y_t, y_v = train_test_split(X_train, y_train,test_size = 0.20)
 print("training model...")
 print("1")
 ridge_model_1.fit(X_train, y_train)
 model_testing(ridge_model_1, X_test = X_v, y_test = y_v)
 print("training model...")
 print("2")
 print("training model...")
 print("3")
 print("TIME:", time.time() - start_time )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1)
<choose_model_class>
def create_submission(model,test = X_test, submission=submission,path="./predictions.csv"):
predictions = model.predict(test)
 predictions = pd.Series(np.exp(predictions)- 1)
 submission['price'] = predictions
submission.to_csv(path, index=False)
 print(submission.describe() )
<train_model><EOS> 
start_time = time.time()
create_submission(ridge_model_1)
 print("TIME:", time.time() - start_time)
 print("TOTAL TIME:", time.time() - Time_0)