Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<data_type_conversions>
np.random.seed(369 )
train_names_unknown_brands_data_test = data_test[data_test['brand_name'] == 'Unknown'][['name','category_name']].\
astype('str' ).values<define_variables>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
train_estimated_brands_train = []
for name, category in tqdm(train_names_unknown_brands_train):
brand = brandfinder(name, category)
 train_estimated_brands_train.append(brand )<define_variables>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
train_estimated_brands_cv = []
for name, category in tqdm(train_names_unknown_brands_cv):
brand = brandfinder(name, category)
 train_estimated_brands_cv.append(brand )<define_variables>
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16
train_estimated_brands_data_test = []
for name, category in tqdm(train_names_unknown_brands_data_test):
brand = brandfinder(name, category)
 train_estimated_brands_data_test.append(brand )<feature_engineering>
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
X_train.loc[X_train['brand_name'] == 'Unknown', 'brand_name'] = train_estimated_brands_train
X_cv.loc[X_cv['brand_name'] == 'Unknown', 'brand_name'] = train_estimated_brands_cv
data_test.loc[data_test['brand_name'] == 'Unknown', 'brand_name'] = train_estimated_brands_data_test<define_variables>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
max_brand = 2500
max_category_name = 1000
max_category = 1000
name_min_df =10
max_item_description_features = 50000<count_values>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
keep_brand = X_train['brand_name'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_brand]
X_train.loc[~X_train['brand_name'].isin(keep_brand), 'brand_name'] = 'Unknown'
X_cv.loc[~X_cv['brand_name'].isin(keep_brand), 'brand_name'] = 'Unknown'
data_test.loc[~data_test['brand_name'].isin(keep_brand), 'brand_name'] = 'Unknown' 
<count_values>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = ['pic_'+str(i)for i in range(train_feats.shape[1])]
keep_category1 = X_train['main_category'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_category]
keep_category2 = X_train['sub_category_1'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_category]
keep_category3 = X_train['sub_category_2'].value_counts().loc[lambda x: x.index != 'Unknown'].index[:max_category]
X_train.loc[~X_train['main_category'].isin(keep_category1), 'main_category'] = 'Unknown'
X_train.loc[~X_train['sub_category_1'].isin(keep_category2), 'sub_category_1'] = 'Unknown'
X_train.loc[~X_train['sub_category_2'].isin(keep_category3), 'sub_category_1'] = 'Unknown'
X_cv.loc[~X_cv['main_category'].isin(keep_category1), 'main_category'] = 'Unknown'
X_cv.loc[~X_cv['sub_category_1'].isin(keep_category2), 'sub_category_1'] = 'Unknown'
X_cv.loc[~X_cv['sub_category_2'].isin(keep_category3), 'sub_category_1'] = 'Unknown'
data_test.loc[~data_test['main_category'].isin(keep_category1), 'main_category'] = 'Unknown'
data_test.loc[~data_test['sub_category_1'].isin(keep_category2), 'sub_category_1'] = 'Unknown'
data_test.loc[~data_test['sub_category_2'].isin(keep_category3), 'sub_category_1'] = 'Unknown'<feature_engineering>
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
tv = TfidfVectorizer(max_features=max_item_description_features,ngram_range=(1, 3),token_pattern=r'(?u)\b\w+\b',stop_words='english')
 tv.fit(X_train['item_description'])
 X_train_item_description_tfidf = tv.transform(X_train['item_description'])
 X_cv_item_description_tfidf = tv.transform(X_cv['item_description'])
 data_test_item_description_tfidf = tv.transform(data_test['item_description'] )<categorify>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = ['pic_'+str(i)for i in range(test_feats.shape[1])]
tv.fit(X_train['name'])
 X_train_name_tfidf = tv.transform(X_train['name'])
 X_cv_name_tfidf = tv.transform(X_cv['name'])
 data_test_name_tfidf = tv.transform(data_test['name'] )<data_type_conversions>
test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats.head()
cv = CountVectorizer(min_df=name_min_df)
 cv.fit(X_train['category_name'].astype('category'))
X_train_category =cv.transform(X_train['category_name'].astype('category'))
X_cv_category =cv.transform(X_cv['category_name'].astype('category'))
data_test_category =cv.transform(data_test['category_name'].astype('category')) 
<categorify>
print('Train')
 train = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/petfinder-adoption-prediction/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/petfinder-adoption-prediction/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/petfinder-adoption-prediction/state_labels.csv")
 print(states.shape)
 target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train = pd.merge(train, train_feats, how='left', on='PetID')
 test = pd.merge(test, test_feats, how='left', on='PetID')
 train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )
lb = LabelBinarizer(sparse_output=True)
 lb.fit(X_train['brand_name'].astype('category'))
X_train_brand = lb.transform(X_train['brand_name'].astype('category'))
X_cv_brand = lb.transform(X_cv['brand_name'])
 data_test_brand = lb.transform(data_test['brand_name'] )<categorify>
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score
X_train_dummies = csr_matrix(pd.get_dummies(X_train[['item_condition_id', 'shipping']], sparse=True ).values)
 X_cv_dummies = csr_matrix(pd.get_dummies(X_cv[['item_condition_id', 'shipping']], sparse=True ).values)
 data_test_dummies = csr_matrix(pd.get_dummies(data_test[['item_condition_id', 'shipping']], sparse=True ).values )<feature_engineering>
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 print("X(tfidf):", X.shape)
 svd = TruncatedSVD(n_components=200)
 svd.fit(X)
 X = svd.transform(X)
 print("X(svd):", X.shape)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(200)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(200)])
 test = pd.concat(( test, X_test), axis=1)
 print("train:", train.shape)
X_train_word_count_item_desc = X_train['item_description'].apply(lambda x: wordCount_with_cleaning(x))
X_cv_word_count_item_desc = X_cv['item_description'].apply(lambda x: wordCount_with_cleaning(x))
data_test_word_count_item_desc = data_test['item_description'].apply(lambda x: wordCount_with_cleaning(x))<count_values>
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores
X_train_word_count_name = X_train['name'].apply(lambda x: wordCount_with_cleaning(x))
X_cv_word_count_name = X_cv['name'].apply(lambda x: wordCount_with_cleaning(x))
data_test_word_count_name = data_test['name'].apply(lambda x: wordCount_with_cleaning(x))<concatenate>
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )
X_train_vectorized = hstack(( X_train_item_description_tfidf,\
X_train_name_tfidf,\
X_train_category,\
X_train_brand,\
X_train_dummies,\
X_train_word_count_item_desc.values.reshape(-1,1),\
X_train_word_count_name.values.reshape(-1,1)) ).tocsr()<concatenate>
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed',
'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac',
'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance',
'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] +\
[col for col in train.columns if col.startswith('pic')or col.startswith('svd')]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 foo = train.dtypes
cat_feature_names = foo[foo == "category"]
cat_features = [train.columns.get_loc(c)for c in train.columns if c in cat_feature_names]
X_cv_vectorized = hstack(( X_cv_item_description_tfidf,\
X_cv_name_tfidf,\
X_cv_category,\
X_cv_brand,\
X_cv_dummies,\
X_cv_word_count_item_desc.values.reshape(-1,1),\
X_cv_word_count_name.values.reshape(-1,1)) ).tocsr()<concatenate>
N_SPLITS = 5
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=N_SPLITS, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], N_SPLITS))
all_coefficients = np.zeros(( N_SPLITS, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/' + str(N_SPLITS))
if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / float(N_SPLITS)
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17,
'early_stop': 600,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
categorical_feature=list(cat_features),
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )
data_test_vectorized = hstack(( data_test_item_description_tfidf,\
data_test_name_tfidf,\
data_test_category,\
data_test_brand,\
data_test_dummies,\
data_test_word_count_item_desc.values.reshape(-1,1),\
data_test_word_count_name.values.reshape(-1,1)) ).tocsr()<drop_column>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )
del(data_train,preprocessed_item_description,preprocessed_test_item_description,X,y)
 del(train_estimated_brands_train,train_estimated_brands_cv,train_estimated_brands_data_test,keep_brand)
 del(data_test_item_description_tfidf,
data_test_name_tfidf,\
data_test_category,\
data_test_brand,\
data_test_dummies,\
data_test_word_count_item_desc,\
data_test_word_count_name)
 del(X_train_item_description_tfidf,
X_train_name_tfidf,\
X_train_category,\
X_train_brand,\
X_train_dummies,\
X_train_word_count_item_desc,\
X_train_word_count_name)
 del(X_cv_item_description_tfidf,
X_cv_name_tfidf,\
X_cv_category,\
X_cv_brand,\
X_cv_dummies,\
X_cv_word_count_item_desc,\
X_cv_word_count_name)
 gc.collect()<compute_test_metric>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)) )<define_variables>
print("True Distribution:")
 print(pd.value_counts(target, normalize=True ).sort_index())
 print("Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index())
 print("Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index() )
Error_dict = {}<init_hyperparams>
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )
params = {
'colsample_bytree': 0.42799939792816927,
'max_depth': 10,
'min_child_samples': 370,
'min_child_weight': 0.01,
'num_leaves': 49,
'reg_lambda': 5,
'subsample': 0.6739316550896339,
'learning_rate':0.1,
'reg_alpha' :0.5,
'boosting_type': 'gbdt',
'objective' : 'regression',
'metric' : 'RMSE',
'verbosity': -1
}
d_train = lgb.Dataset(X_train_vectorized, label=Y_train)
 d_valid = lgb.Dataset(X_cv_vectorized, label=Y_cv)
 watchlist = [d_train, d_valid]
model_1 = lgb.train(params, train_set=d_train,valid_sets=watchlist,num_boost_round=2000,verbose_eval=200,early_stopping_rounds=100)
<predict_on_test>
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, [r[0] for r in results['train']])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()
lgbm_1_pred_train=model_1.predict(X_train_vectorized)
 lgbm_1_pred_cv=model_1.predict(X_cv_vectorized)
 lgbm_1_pred_data_test=model_1.predict(data_test_vectorized )<compute_test_metric>
submission.to_csv('submission.csv', index=False )
<compute_test_metric><EOS> 
submission.to_csv('submission.csv', index=False )