Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<train_model>
np.random.seed(369 )
%%time
preprocess(X_train, train=True)
 printtime("preprocess training data done" )<feature_engineering>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
cat1_cv = CountVectorizer(token_pattern='.+')
 cat2_cv = CountVectorizer(token_pattern='.+')
 cat3_cv = CountVectorizer(token_pattern='.+')
 brand_cv = CountVectorizer(token_pattern='.+')
 condition_cv = CountVectorizer(token_pattern='.+')
 shipping_cv = CountVectorizer(token_pattern='.+')
 has_brand_cv = CountVectorizer(token_pattern='.+')
 cat1_cond_cv = CountVectorizer(token_pattern='.+')
 cat2_cond_cv = CountVectorizer(token_pattern='.+')
 cat3_cond_cv = CountVectorizer(token_pattern='.+' )<create_dataframe>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
mapper = sklearn_pandas.DataFrameMapper([ 
('name', TfidfVectorizer(ngram_range=(1, 2))), 
('cat1', cat1_cv), 
('cat2', cat2_cv), 
('cat3', cat3_cv), 
('cat1_cond', cat1_cond_cv), 
('cat2_cond', cat2_cond_cv), 
('cat3_cond', cat3_cond_cv), 
('brand_name', brand_cv), 
('item_description', TfidfVectorizer(ngram_range=(1, 2), max_features=100000)) , 
('item_condition_id', condition_cv), 
('shipping', shipping_cv), 
('description_length', None), 
('name_length', None), 
('has_brand', has_brand_cv)
 ], sparse=True )<choose_model_class>
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
model_ridge = Pipeline([ 
('map', mapper), 
('reg', Ridge(alpha=3.0))
] )<train_model>
def get_sentiment(ids, path):
dic = {
"doc_sent_mag": [],
"doc_sent_score": []
}
for pet in ids:
try:
with open(path + pet + '.json', 'r')as f:
sentiment = json.load(f)
 dic["doc_sent_mag"].append(sentiment['documentSentiment']['magnitude'])
 dic["doc_sent_score"].append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
dic["doc_sent_mag"].append(np.nan)
 dic["doc_sent_score"].append(np.nan)
 return dic
def get_meta(ids, path, n):
dic = {
"vertex_xs": [],
"vertex_ys": [],
"bounding_confidences": [],
"bounding_importance_fracs": [],
"dominant_blues": [],
"dominant_greens": [],
"dominant_reds": [],
"dominant_pixel_fracs": [],
"dominant_scores": [],
"mean_red": [],
"mean_blue": [],
"mean_green": [],
"label_descriptions": [],
"label_scores": []
}
for pet in ids:
try:
with open(path+ pet + '-{}.json'.format(str(n)) , 'r')as f:
data = json.load(f)
 dic["vertex_xs"].append(data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x'])
 dic["vertex_ys"].append(data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y'])
 dic["bounding_confidences"].append(data['cropHintsAnnotation']['cropHints'][0]['confidence'])
 dic["bounding_importance_fracs"].append(data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1))
try:
dic["dominant_blues"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue'])
 except:
dic["dominant_blues"].append(0)
 try:
dic["dominant_greens"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green'])
 except:
dic["dominant_greens"].append(0)
 try:
dic["dominant_reds"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red'])
 except:
dic["dominant_reds"].append(0)
 dic["dominant_pixel_fracs"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction'])
 dic["dominant_scores"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score'])

if data.get('labelAnnotations'):
dic["label_descriptions"].append(data['labelAnnotations'][0]['description'])
 dic["label_scores"].append(data['labelAnnotations'][0]['score'])
 else:
dic["label_descriptions"].append('nothing')
 dic["label_scores"].append(np.nan)
 except FileNotFoundError:
dic["vertex_xs"].append(np.nan)
 dic["vertex_ys"].append(np.nan)
 dic["bounding_confidences"].append(np.nan)
 dic["bounding_importance_fracs"].append(np.nan)
 dic["dominant_blues"].append(np.nan)
 dic["dominant_greens"].append(np.nan)
 dic["dominant_reds"].append(np.nan)
 dic["dominant_pixel_fracs"].append(np.nan)
 dic["dominant_scores"].append(np.nan)
 dic["label_descriptions"].append('nothing')
 dic["label_scores"].append(np.nan)
 return dic
def replace_nan(train, test, replace_dic):
for col, value in replace_dic.items() :
train[col] = train[col].replace(value, np.nan)
 test[col] = test[col].replace(value, np.nan)
 return train, test
def get_color(x):
result = [0 for i in range(7)]
for i in range(1, 8):
if i in list(x.values):
result[i-1] = 1
return pd.Series(result )
%%time
model_ridge.fit(X_train, y_train )<categorify>
%%time
train = pd.read_csv(".. /input/train/train.csv")
 test = pd.read_csv(".. /input/test/test.csv" )
len(mapper.transformed_names_ )<train_model>
fe_input_path = ".. /input/"
n_meta = 1
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True)
 color_result_col = ["color{}".format(i)for i in range(1, 8)]
color_pick_col = ["Color1", "Color2", "Color3"]
train_color = pd.DataFrame(train[color_pick_col].apply(get_color, axis=1 ).values, columns=color_result_col)
 test_color = pd.DataFrame(test[color_pick_col].apply(get_color, axis=1 ).values, columns=color_result_col)
 train = pd.concat(( train, train_color), axis=1)
 test = pd.concat(( test, test_color), axis=1)
 train.drop(color_pick_col, axis=1, inplace=True)
 test.drop(color_pick_col, axis=1, inplace=True )
%%time
wb_name = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29,
"norm": None,
"tf": 'binary',
"idf": None,
}), procs=8)
 wb_name.dictionary_freeze = True
wb_name.fit(X_train['name'] )<train_model>
%%time
dic = get_sentiment(train_id, join(fe_input_path, 'train_sentiment/'))
train.loc[:, 'doc_sent_mag'] = dic["doc_sent_mag"]
train.loc[:, 'doc_sent_score'] = dic["doc_sent_score"]
dic = get_sentiment(test_id, join(fe_input_path, 'test_sentiment/'))
test.loc[:, 'doc_sent_mag'] = dic["doc_sent_mag"]
test.loc[:, 'doc_sent_score'] = dic["doc_sent_score"]
%%time
wb_desc = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28,
"norm": "l2",
"tf": 1.0,
"idf": None
}), procs=8)
 wb_desc.dictionary_freeze = True
wb_desc.fit(X_train['item_description'] )<categorify>
for i in range(1, n_meta+1):
dic = get_meta(train_id, join(fe_input_path, 'train_metadata/'), i)
 train.loc[:, 'vertex_x_{}'.format(str(i)) ] = dic["vertex_xs"]
train.loc[:, 'vertex_y_{}'.format(str(i)) ] = dic["vertex_ys"]
train.loc[:, 'bounding_confidence_{}'.format(str(i)) ] = dic["bounding_confidences"]
train.loc[:, 'bounding_importance_{}'.format(str(i)) ] = dic["bounding_importance_fracs"]
train.loc[:, 'dominant_blue_{}'.format(str(i)) ] = dic["dominant_blues"]
train.loc[:, 'dominant_green_{}'.format(str(i)) ] = dic["dominant_greens"]
train.loc[:, 'dominant_red_{}'.format(str(i)) ] = dic["dominant_reds"]
train.loc[:, 'dominant_pixel_frac_{}'.format(str(i)) ] = dic["dominant_pixel_fracs"]
train.loc[:, 'dominant_score_{}'.format(str(i)) ] = dic["dominant_scores"]
train.loc[:, 'label_description_{}'.format(str(i)) ] = dic["label_descriptions"]
train.loc[:, 'label_score_{}'.format(str(i)) ] = dic["label_scores"]
dic = get_meta(test_id, join(fe_input_path, 'test_metadata/'), i)
 test.loc[:, 'vertex_x_{}'.format(str(i)) ] = dic["vertex_xs"]
test.loc[:, 'vertex_y_{}'.format(str(i)) ] = dic["vertex_ys"]
test.loc[:, 'bounding_confidence_{}'.format(str(i)) ] = dic["bounding_confidences"]
test.loc[:, 'bounding_importance_{}'.format(str(i)) ] = dic["bounding_importance_fracs"]
test.loc[:, 'dominant_blue_{}'.format(str(i)) ] = dic["dominant_blues"]
test.loc[:, 'dominant_green_{}'.format(str(i)) ] = dic["dominant_greens"]
test.loc[:, 'dominant_red_{}'.format(str(i)) ] = dic["dominant_reds"]
test.loc[:, 'dominant_pixel_frac_{}'.format(str(i)) ] = dic["dominant_pixel_fracs"]
test.loc[:, 'dominant_score_{}'.format(str(i)) ] = dic["dominant_scores"]
test.loc[:, 'label_description_{}'.format(str(i)) ] = dic["label_descriptions"]
test.loc[:, 'label_score_{}'.format(str(i)) ] = dic["label_scores"]
mask_name = None
mask_desc = None
mask_total = None
def transform_fm(data, train=False):
global mask_name
global mask_desc
global mask_total
X_cat1 = cat1_cv.transform(data['cat1'])
 X_cat2 = cat2_cv.transform(data['cat2'])
 X_cat3 = cat3_cv.transform(data['cat3'])
 X_cat1_cond = cat1_cond_cv.transform(data['cat1_cond'])
 X_cat2_cond = cat2_cond_cv.transform(data['cat2_cond'])
 X_cat3_cond = cat3_cond_cv.transform(data['cat3_cond'])
 X_brand = brand_cv.transform(data['brand_name'])
 X_condition = shipping_cv.transform(data['shipping'])
 X_shipping = condition_cv.transform(data['item_condition_id'])
 X_name_length = csr_matrix(data['name_length'] ).transpose()
X_desc_length = csr_matrix(data['description_length'] ).transpose()
X_name = wb_name.transform(data['name'])
 print(X_name.shape)
 if train:
mask_name = np.array(np.clip(X_name.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X_name = X_name[:, mask_name]
print(X_name.shape)
 X_desc = wb_desc.transform(data['item_description'])
 print(X_desc.shape)
 if train:
mask_desc = np.array(np.clip(X_desc.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X_desc = X_desc[:, mask_desc]
print(X_desc.shape)
 X_has_brand = has_brand_cv.transform(data['has_brand'])
 X = hstack(( X_name, X_desc, X_cat1, X_cat2, X_cat3, X_cat1_cond, X_cat2_cond, X_cat3_cond, X_brand, X_condition, X_shipping, X_name_length, X_desc_length, X_has_brand)).tocsr()
print(X.shape)
 if train:
mask_total = np.array(np.clip(X.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X = X[:, mask_total]
print(X.shape)
 return X<categorify>
eng_stopwords = set(stopwords.words("english"))
remove_punctuation_map = dict(( ord(char), ' ')for char in string.punctuation)
 stemmer = nltk.stem.porter.PorterStemmer()
def stem_tokens(tokens):
lst = [stemmer.stem(item)for item in tokens]
return ' '.join(lst)
 def get_textfeats(df, col, flag=True):
df[col] = df[col].fillna('none' ).astype(str)
 df[col] = df[col].str.lower()
df[col] = df[col].apply(lambda x: stem_tokens(nltk.word_tokenize(x.translate(remove_punctuation_map))))
return df
def load_text(train, test):
train = get_textfeats(train, "Description")
 test = get_textfeats(test, "Description")
 train_desc = train['Description'].values
test_desc = test['Description'].values
train_corpus = [text_to_word_sequence(text)for text in tqdm(train_desc)]
test_corpus = [text_to_word_sequence(text)for text in tqdm(test_desc)]
return train_corpus, test_corpus
def get_result(corpus, model):
result = []
for text in corpus:
n_skip = 0
for n_w, word in enumerate(text):
try:
vec_ = model.wv[word]
except:
n_skip += 1
continue
if n_w == 0:
vec = vec_
else:
vec = vec + vec_
vec = vec /(n_w - n_skip + 1)
 result.append(vec)
 return result
train_corpus, test_corpus = load_text(train, test)
 model = word2vec.Word2Vec(train_corpus+test_corpus, size=200, window=10, max_vocab_size=50000, seed=0)
 train_result = get_result(train_corpus, model)
 test_result = get_result(test_corpus, model)
 w2v_cols = ["wv{}".format(i)for i in range(1, 201)]
train_result = pd.DataFrame(train_result)
 train_result.columns = w2v_cols
test_result = pd.DataFrame(test_result)
 test_result.columns = w2v_cols
train = pd.concat(( train, train_result), axis=1)
 test = pd.concat(( test, test_result), axis=1 )
%%time
X_train_fm = transform_fm(X_train, train=True )<train_model>
%%time
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )
%%time
model_fm = FM_FTRL(alpha=0.01, beta=0.01, L1=0.00001, L2=0.1, D=X_train_fm.shape[1], alpha_fm=0.01, L2_fm=0.0, init_fm=0.01,
D_fm=200, e_noise=0.0001, iters=15, inv_link="identity", threads=4)
 model_fm.fit(X_train_fm, y_train )<predict_on_test>
replace_dic = {
"Gender": 3,
"MaturitySize": 0,
"FurLength": 0,
"MaturitySize": 0,
"Vaccinated": 3,
"Dewormed": 3,
"Sterilized": 3,
"Health": 0
}
train, test = replace_nan(train, test, replace_dic)
 cat_cols = ['Type', 'Breed1', 'Breed2', 'Gender', 'MaturitySize',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized', 'Health',
'State', 'label_description_1']
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()
def predict(X_test, X_test_fm):
y_ridge = model_ridge.predict(X_test)
 y_fm = model_fm.predict(X_test_fm)
 w = 0.20
y = w*y_ridge+(1-w)*y_fm
y = np.clip(np.expm1(y), MIN_PRICE, MAX_PRICE)
 return y<save_to_csv>
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )
%%time
if not develop:
printtime("Processing test data")
 chunk = 1
for X_test in pd.read_csv('.. /input/test.tsv', sep='\t', chunksize=PREDICT_BATCH_SIZE):
preprocess(X_test)
 X_test_fm = transform_fm(X_test)
 y = predict(X_test, X_test_fm)
 results = pd.DataFrame({'test_id': X_test['test_id'], 'price': y}, columns=['test_id', 'price'])
 mode = 'w' if chunk == 1 else 'a'
write_header = chunk == 1
results.to_csv('submission_rige_ftrl.csv', index=False, header=write_header, mode=mode)
 printtime("Processed test data batch " + str(chunk))
chunk += 1
else:
preprocess(X_test)
 printtime("preprocess test data done")
 X_test_fm = transform_fm(X_test)
 printtime("prepare FM data done")
 y = predict(X_test, X_test_fm)
 printtime("predictions done" )<compute_test_metric>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )
if develop:
y_test_exp = np.expm1(y_test.values)
 print("RMSLE Ensemble", metrics.mean_squared_log_error(y_test_exp, y)** 0.5)
<train_model>
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )
printtime("kernel done" )<set_options>
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )
%matplotlib inline
seed = 90<load_from_csv>
quadratic_weighted_kappa(target, train_predictions )
Time_0 = time.time()
train = pd.read_csv('.. /input/train.tsv',sep='\t')
 test = pd.read_csv('.. /input/test.tsv',sep='\t' )<prepare_x_and_y>
rmse(target, [r[0] for r in results['train']] )
y_train = train['log_price'] = np.log(( train['price'] + 1))<categorify>
submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()
def handle_missing(dataset):
dataset['category_name'].fillna(value="NA/NA/NA", inplace=True)
 dataset['brand_name'].fillna(value="missing", inplace=True)
 dataset['item_description'].fillna(value="missing", inplace=True)
 return(dataset)
 def split_cat(dataset):
dataset['cat1'], dataset['cat2'], dataset['cat3'] = zip(*dataset['category_name'].str.split("/",2))
return dataset
def label_maker(dataset):
lb = LabelBinarizer(sparse_output=True)
 cat1 = lb.fit_transform(dataset['cat1'])
 cat2 = lb.fit_transform(dataset['cat2'])
 cat3 = lb.fit_transform(dataset['cat3'])
 brand_name = lb.fit_transform(dataset['brand_name'])
 del lb
return cat1,cat2,cat3,brand_name
def get_dums(dataset):
X_dummies = csr_matrix(pd.get_dummies(dataset[['item_condition_id', 'shipping']],
sparse=True ).values)
 return X_dummies
def text_processing(dataset):
MIN_DF_COUNT = 10
MAX_DF_COUNT = 10000
cv = CountVectorizer(min_df = MIN_DF_COUNT, max_df = MAX_DF_COUNT)
 name = cv.fit_transform(dataset['name'])
 MIN_DF_TF = 10
MAX_DF_TF = 51000
MAX_FEATURES_TF = 51000
tv = TfidfVectorizer(max_features=MAX_FEATURES_TF,
min_df = MIN_DF_TF,
max_df = MAX_DF_TF,
ngram_range=(1, 3),
stop_words='english')
 description = tv.fit_transform(dataset['item_description'])
 del cv, tv
return name, description 
<create_dataframe>
submission.to_csv('submission.csv', index=False )
<categorify><EOS> 
submission.to_csv('submission.csv', index=False )