Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<load_from_csv>
py.init_notebook_mode(connected=True)
 %matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
%%time
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape )<prepare_x_and_y>
def prepare_data(df):
def _clean(text):
return re.sub(r'[^\w\s]','',text)
 def _lower(text):
return text.lower()
for column in ['name', 'brand_name', 'item_description', 'item_condition_id']:
df[column] = df[column].astype(str ).apply(_clean ).apply(_lower)
 df['category_name'] = df['category_name'].astype(str ).apply(_lower)
 def _split_cat(text):
cats = text.split("/")
 if len(cats)>=3:
return cats[0:3]
else: return("No Label", "No Label", "No Label")
 df['general_cat'], df['subcat_1'], df['subcat_2'] = zip(*df['category_name'].apply(lambda x: _split_cat(x)))
 def _length(text):
tokens = re.sub(r'[^\w\s]','',text ).lower()
return len(tokens.split())
 df['desc_len'] = df['item_description'].apply(_length)
 def _concat_(df):
text = ""
for col in ['name', 'general_cat', 'subcat_1', 'subcat_2', 'brand_name', 'item_condition_id']:
text = text +(df[col])+ " "
return text
df['other_info'] = df.loc[:, ['name', 'general_cat', 'subcat_1', 'subcat_2', 'brand_name', 'item_condition_id']].apply(_concat_, axis=1)
 df['other_info_len'] = df['other_info'].apply(_length)
 return df
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )<feature_engineering>
df = pd.read_csv(".. /input/train.tsv", sep='\t')
 df.head()
df = prepare_data(df )
%%time
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score<train_on_grid>
corpus = []
for row in range(len(df)) :
for column in ['other_info', 'item_description']:
tokens =(df.loc[row, column].split())
 corpus.append(tokens)
 embedding_size = 50
vocab_size = 50000
w2v_model = Word2Vec(corpus, size=embedding_size, min_count=0, max_vocab_size=vocab_size, workers=8, negative=5, sg=1, iter=1,
batch_words=10000, sorted_vocab=1)
 words = list(w2v_model.wv.vocab)
 dictionary = {}
word_embeddings = np.zeros(( len(words), embedding_size), dtype=float)
 row = 0
for word in words:
dictionary[word] = len(dictionary)
 word_embeddings[row, :] = w2v_model.wv.get_vector(word)
 row += 1
reverse_dictionary = dict(zip(dictionary.values() , dictionary.keys()))
pickle.dump(word_embeddings, open("word_embeddings.p", "wb"))
del w2v_model, words, corpus
%%time
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )<prepare_x_and_y>
word_embeddings = word_embeddings.astype("float32" )
%%time
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<drop_column>
max_D, max_O = 75, 20
def to_array(df):
def _to_array(df, column, max_length):
indices = np.zeros(( len(df), max_length))
length = np.zeros(( len(df),1))
for row in range(len(df)) :
tokens = df.loc[row, column].split()
i = 0
for token in tokens:
if token in dictionary:
indices[row, i] = dictionary[token]
i += 1
if i == max_length: break
length[row] = i
return indices, length
desc_ind, desc_len = _to_array(df, 'item_description', max_D)
 other_ind, other_len = _to_array(df, 'other_info', max_O)
 data = np.hstack(( desc_ind, other_ind, desc_len, other_len))
if 'price' in df.columns:
log_price = np.log(df['price']+1 ).values.reshape(( len(df),1)).astype('float32')
 data = np.hstack(( data, log_price))
return data
data = to_array(df)
 del df
data.shape
%%time
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )<data_type_conversions>
def train_valid_split(array):
train_id_1 = np.arange(len(array)//3)* 3
train_id_2 = np.arange(len(array)//3)* 3 + 1
valid_id = np.arange(len(array)//3)* 3 + 2
train_id = np.hstack(( train_id_1, train_id_2))
train = array[train_id, :]
valid = array[valid_id, :]
return train, valid
train, valid = train_valid_split(data)
 train.shape, valid.shape
del data
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(120)]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<find_best_model_class>
def get_batch(array, start, size):
if start + size > len(array):
end = len(array)
 else: end = start + size
batch = array[start:end, :]
if end == len(array): start = 0
else: start = start + size
return batch, start
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 11,
'learning_rate': 0.01,
'bagging_fraction': 0.9,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
word_dim = 50
assert word_dim == embedding_size
num_hidden = 50
learning_rate = 0.001
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
tf.reset_default_graph()
graph = tf.Graph()
with graph.as_default() :
with tf.device('/gpu:0'):
desc_indices = tf.placeholder(tf.int32, shape=[None, max_D])
 info_indices = tf.placeholder(tf.int32, shape=[None, max_O])
 desc_seq_length = tf.placeholder(tf.int8, [None,])
 info_seq_length = tf.placeholder(tf.int8, [None,])
 log_price = tf.placeholder(tf.float32, [None,1])
 embeddings = tf.Variable(word_embeddings)
 x_desc = tf.nn.embedding_lookup(embeddings, desc_indices)
 x_desc = tf.unstack(x_desc, max_D, 1)
 x_info = tf.nn.embedding_lookup(embeddings, info_indices)
 x_info = tf.unstack(x_info, max_O, 1)
 gru_cell = tf.nn.rnn_cell.GRUCell(num_hidden)
 outputs_desc, state_desc = tf.nn.static_rnn(gru_cell, x_desc, sequence_length=desc_seq_length, dtype=tf.float32)
 outputs_info, state_info = tf.nn.static_rnn(gru_cell, x_info, sequence_length=info_seq_length, dtype=tf.float32)
 weights_desc = tf.Variable(tf.random_normal([num_hidden,1]))
weights_info = tf.Variable(tf.random_normal([num_hidden,1]))
biases = tf.Variable(tf.random_normal([1,1]))
pred = tf.matmul(state_desc, weights_desc)+ tf.matmul(state_info, weights_info)+ biases
error = log_price - pred
loss = tf.sqrt(tf.reduce_mean(tf.square(error)))
 train_op = tf.train.AdamOptimizer(learning_rate=learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
saver = tf.train.Saver()
print("flow graph constructed" )
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
def get_feed_dict(batch):
if batch.shape[1] == 98:
feed_dict={ desc_indices: batch[:, 0:75],
info_indices: batch[:, 75:95],
desc_seq_length: batch[:, 95],
info_seq_length: batch[:, 96],
log_price: batch[:, 97:98]}
else:
feed_dict={ desc_indices: batch[:, 0:75],
info_indices: batch[:, 75:95],
desc_seq_length: batch[:, 95],
info_seq_length: batch[:, 96],
}
return feed_dict
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
def predict(session, data, batch_size):
start = 0
for step in range(len(data)//batch_size + 1):
batch, start = get_batch(data, start, batch_size)
 pred_ = sess.run(pred, feed_dict=get_feed_dict(batch))
if step == 0:
pred_all = np.array(pred_)
 else:
pred_all = np.vstack(( pred_all, pred_))
assert len(pred_all)== len(data)
 return pred_all
quadratic_weighted_kappa(target, train_predictions )<compute_test_metric>
train_start, valid_start, time_initial = 0, 0, time()
batch_size, epochs = 500, 2
no_batches = len(train)//batch_size + 1
avg_loss, time_0 = 0, time()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True), graph=graph)as sess:
init.run()
for epoch in range(epochs):
for step in range(no_batches):
if step % 1000 == 0: print(step)
 batch, train_start = get_batch(train, train_start, batch_size)
 _, loss_step = sess.run([train_op, loss], feed_dict=get_feed_dict(batch))
avg_loss += loss_step
avg_loss /= no_batches
print("epoch: %2d, average loss: %.3f, time: %2.2f"
%(epoch, avg_loss, time() -time_0))
avg_loss, time_0 = 0, time()
valid_pred = predict(sess, valid, batch_size)
 valid_log_price = valid[:, 97:98]
valid_rsme = np.sqrt(np.mean(( valid_pred - valid_log_price)**2))
print("validation rsme: %.4f" %valid_rsme)
 save_path = saver.save(sess,"./params.ckpt")
 print("saved parameters to: ", save_path)
 print("done training")
 print("total training time is: %4d" %(time() - time_initial))
rmse(target, [r[0] for r in results['train']] )<prepare_output>
del train, valid
test = pd.read_csv(".. /input/test_stg2.tsv", sep='\t')
 test = prepare_data(test)
 test = to_array(test)
 print(test.shape )
submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True), graph=graph)as sess:
saver.restore(sess, "./params.ckpt")
 test_pred = predict(sess, test, batch_size)
 prices = np.exp(test_pred)- 1
assert len(test_pred)== len(test)
 out = pd.DataFrame(prices)
 out = out.reset_index().rename(columns={"index":"test_id", 0:"price"} )
<save_to_csv><EOS> 
out.to_csv("./out.csv", index=False)
 print("output saved to csv" )