Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<define_variables>
import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix, hstack
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.linear_model import Ridge
from sklearn import metrics
from sklearn import preprocessing
import sklearn_pandas
import random
import wordbatch
from wordbatch.extractors import WordBag
from wordbatch.models import FTRL, FM_FTRL
import re
import gc
import time
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}<define_variables>
develop = False
develop_size = 0
NUM_BRANDS = 4500
NUM_CATEGORIES = 1200
MIN_PRICE = 3
MAX_PRICE = 2000
PREDICT_BATCH_SIZE = 350000
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
train = pd.read_csv('.. /input/train.tsv', sep='\t')
 if develop_size > 0:
train = train.sample(develop_size, random_state=42)
 printtime("train loaded" )
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, hyphens_dict)
 normalized_word = normalize_by_dictionary(normalized_word, merged_dict)
 normalized_word = normalize_by_dictionary(normalized_word, misspellings_dict)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result 
<load_from_csv>
def ngrams(text, ngram_range):
words = text.split()
for wordcount in range(ngram_range[0], ngram_range[1]+1):
for i in range(0, len(words)-wordcount+1):
yield " ".join(words[i:i+wordcount])
 def match_brand_name(row, top_brands):
if row['brand_name'] != "unknown":
return row['brand_name']
if row['name'] in top_brands:
return row['name']
for subname in reversed(list(ngrams(row['name'],(1, 3)))) :
if subname in top_brands:
return subname
return row['brand_name']
def fill_brand(data, top_brands):
data['brand_name'] = data[['brand_name', 'name']].apply(lambda x : match_brand_name(x, top_brands), axis=1)
 def split_cat(text):
try:
return text.split("/")
 except:
return("unknown", "unknown", "unknown")
 non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1] )
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
def calc_stats(data):
def topn_values(col, n):
return set(col.value_counts().loc[lambda x: x.index != 'unknown'].index[:n].values)
 return {
'top_brands': topn_values(data['brand_name'], NUM_BRANDS),
'top_cat1': topn_values(data['cat1'], NUM_CATEGORIES),
'top_cat2': topn_values(data['cat2'], NUM_CATEGORIES),
'top_cat3': topn_values(data['cat3'], NUM_CATEGORIES),
}
%%time
labels, x_train, x_test = read_data_files(train_filepath, test_filepath )<string_transform>
stats = None
def preprocess(data, train=False):
global stats
data['cat1'], data['cat2'], data['cat3'] = zip(*data['category_name'].apply(lambda x: split_cat(x)))
 data['name'].fillna("unknown", inplace=True)
 data['item_description'] = data['item_description']\ 
.fillna("unknown")\ 
.apply(lambda x: x.replace('[rm]', '')) \ 
.apply(lambda x: x.replace('No description yet', 'unknown'))
data['has_brand'] = data['brand_name'].apply(lambda x: 'no_brand' if pd.isnull(x)else 'has_brand')
 data['brand_name'].fillna("unknown", inplace=True)
 data['item_condition_id'] = data['item_condition_id'].astype(str ).fillna("unknown")
 data['shipping'] = data['item_condition_id'].astype(str ).fillna("unknown")
 data['cat1_cond'] = data['cat1'] + "_" + data['item_condition_id']
data['cat2_cond'] = data['cat2'] + "_" + data['item_condition_id']
data['cat3_cond'] = data['cat3'] + "_" + data['item_condition_id']
unit_regex = r'(\d+)[\s-]([a-z]{2} )(\s)'
unit_repl = r'\1\2\3'
karats_regex = r'(\d )([\s-]? )(karat|karats|carat|carats|kt )([^\w])'
karats_repl = r'\1k\4'
data['name'] = data['name'].str.replace(unit_regex, unit_repl)
 data['item_description'] = data['item_description'].str.replace(unit_regex, unit_repl)
 data['name'] = data['name'].str.replace(karats_regex, karats_repl)
 data['item_description'] = data['item_description'].str.replace(karats_regex, karats_repl)
 if train:
stats = calc_stats(data)
 fill_brand(data, stats['top_brands'])
 data.loc[~data['brand_name'].isin(stats['top_brands']), 'brand_name'] = 'unknown'
data.loc[~data['cat1'].isin(stats['top_cat1']), 'cat1'] = 'unknown'
data.loc[~data['cat2'].isin(stats['top_cat2']), 'cat2'] = 'unknown'
data.loc[~data['cat3'].isin(stats['top_cat3']), 'cat3'] = 'unknown'
data['description_length'] = preprocessing.scale(data['item_description'].apply(lambda x: len(x.split())).astype(float))
data['name_length'] = preprocessing.scale(data['name'].apply(lambda x: len(x.split())).astype(float))
data['cat1_cat'] = data['cat1'].astype('category')
 data['cat2_cat'] = data['cat2'].astype('category')
 data['cat3_cat'] = data['cat3'].astype('category')
 data['item_condition_id_cat'] = data['item_condition_id'].astype('category' )
%%time
fileObject = open('.. /input/cleaning-dictionaries/tokenizer','rb')
 tokenizer = pickle.load(fileObject)
 x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<load_pretrained>
train.drop(train[train['price'] < MIN_PRICE].index, inplace=True)
 printtime("dropped erroneous prices < 3")
 X_train = train
y_train = np.log1p(train['price'])
 X_test = []
y_test = []
if develop:
X_train, X_test, y_train, y_test = train_test_split(train, y_train, random_state=42, test_size=0.1)
 X_train.is_copy = False
X_test.is_copy = False
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )<split>
%%time
preprocess(X_train, train=True)
 printtime("preprocess training data done" )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1)
<choose_model_class>
cat1_cv = CountVectorizer(token_pattern='.+')
 cat2_cv = CountVectorizer(token_pattern='.+')
 cat3_cv = CountVectorizer(token_pattern='.+')
 brand_cv = CountVectorizer(token_pattern='.+')
 condition_cv = CountVectorizer(token_pattern='.+')
 shipping_cv = CountVectorizer(token_pattern='.+')
 has_brand_cv = CountVectorizer(token_pattern='.+')
 cat1_cond_cv = CountVectorizer(token_pattern='.+')
 cat2_cond_cv = CountVectorizer(token_pattern='.+')
 cat3_cond_cv = CountVectorizer(token_pattern='.+' )
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
mapper = sklearn_pandas.DataFrameMapper([ 
('name', TfidfVectorizer(ngram_range=(1, 2))), 
('cat1', cat1_cv), 
('cat2', cat2_cv), 
('cat3', cat3_cv), 
('cat1_cond', cat1_cond_cv), 
('cat2_cond', cat2_cond_cv), 
('cat3_cond', cat3_cond_cv), 
('brand_name', brand_cv), 
('item_description', TfidfVectorizer(ngram_range=(1, 2), max_features=100000)) , 
('item_condition_id', condition_cv), 
('shipping', shipping_cv), 
('description_length', None), 
('name_length', None), 
('has_brand', has_brand_cv)
 ], sparse=True )
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )<save_to_csv>
model_ridge = Pipeline([ 
('map', mapper), 
('reg', Ridge(alpha=3.0))
] )
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<set_options>
%%time
model_ridge.fit(X_train, y_train )
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<define_variables>
len(mapper.transformed_names_ )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<load_from_csv>
%%time
wb_name = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29,
"norm": None,
"tf": 'binary',
"idf": None,
}), procs=8)
 wb_name.dictionary_freeze = True
wb_name.fit(X_train['name'] )
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 y = train_df[list_classes].values
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = y
x_test = test_df[TEXT_COLUMN].astype(str )<categorify>
%%time
wb_desc = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28,
"norm": "l2",
"tf": 1.0,
"idf": None
}), procs=8)
 wb_desc.dictionary_freeze = True
wb_desc.fit(X_train['item_description'] )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<string_transform>
mask_name = None
mask_desc = None
mask_total = None
def transform_fm(data, train=False):
global mask_name
global mask_desc
global mask_total
X_cat1 = cat1_cv.transform(data['cat1'])
 X_cat2 = cat2_cv.transform(data['cat2'])
 X_cat3 = cat3_cv.transform(data['cat3'])
 X_cat1_cond = cat1_cond_cv.transform(data['cat1_cond'])
 X_cat2_cond = cat2_cond_cv.transform(data['cat2_cond'])
 X_cat3_cond = cat3_cond_cv.transform(data['cat3_cond'])
 X_brand = brand_cv.transform(data['brand_name'])
 X_condition = shipping_cv.transform(data['shipping'])
 X_shipping = condition_cv.transform(data['item_condition_id'])
 X_name_length = csr_matrix(data['name_length'] ).transpose()
X_desc_length = csr_matrix(data['description_length'] ).transpose()
X_name = wb_name.transform(data['name'])
 print(X_name.shape)
 if train:
mask_name = np.array(np.clip(X_name.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X_name = X_name[:, mask_name]
print(X_name.shape)
 X_desc = wb_desc.transform(data['item_description'])
 print(X_desc.shape)
 if train:
mask_desc = np.array(np.clip(X_desc.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X_desc = X_desc[:, mask_desc]
print(X_desc.shape)
 X_has_brand = has_brand_cv.transform(data['has_brand'])
 X = hstack(( X_name, X_desc, X_cat1, X_cat2, X_cat3, X_cat1_cond, X_cat2_cond, X_cat3_cond, X_brand, X_condition, X_shipping, X_name_length, X_desc_length, X_has_brand)).tocsr()
print(X.shape)
 if train:
mask_total = np.array(np.clip(X.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X = X[:, mask_total]
print(X.shape)
 return X
%%time
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<concatenate>
%%time
X_train_fm = transform_fm(X_train, train=True )
%%time
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<split>
%%time
model_fm = FM_FTRL(alpha=0.01, beta=0.01, L1=0.00001, L2=0.1, D=X_train_fm.shape[1], alpha_fm=0.01, L2_fm=0.0, init_fm=0.01,
D_fm=200, e_noise=0.0001, iters=15, inv_link="identity", threads=4)
 model_fm.fit(X_train_fm, y_train )
X_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train, test_size = 0.1)
<train_model>
def predict(X_test, X_test_fm):
y_ridge = model_ridge.predict(X_test)
 y_fm = model_fm.predict(X_test_fm)
 w = 0.20
y = w*y_ridge+(1-w)*y_fm
y = np.clip(np.expm1(y), MIN_PRICE, MAX_PRICE)
 return y
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 X_train,
Y_train,
validation_data =(X_valid, Y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 val_preds_3 = model.predict(X_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(Y_valid[:,i], val_preds_3[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5')
<save_to_csv>
%%time
if not develop:
printtime("Processing test data")
 chunk = 1
for X_test in pd.read_csv('.. /input/test.tsv', sep='\t', chunksize=PREDICT_BATCH_SIZE):
preprocess(X_test)
 X_test_fm = transform_fm(X_test)
 y = predict(X_test, X_test_fm)
 results = pd.DataFrame({'test_id': X_test['test_id'], 'price': y}, columns=['test_id', 'price'])
 mode = 'w' if chunk == 1 else 'a'
write_header = chunk == 1
results.to_csv('submission_rige_ftrl.csv', index=False, header=write_header, mode=mode)
 printtime("Processed test data batch " + str(chunk))
chunk += 1
else:
preprocess(X_test)
 printtime("preprocess test data done")
 X_test_fm = transform_fm(X_test)
 printtime("prepare FM data done")
 y = predict(X_test, X_test_fm)
 printtime("predictions done" )
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<import_modules>
if develop:
y_test_exp = np.expm1(y_test.values)
 print("RMSLE Ensemble", metrics.mean_squared_log_error(y_test_exp, y)** 0.5)
<load_from_csv><EOS> 
printtime("kernel done" )