Kaggle competition: PetFinder.my Adoption Prediction
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
import gc
import glob
import os
import json
import matplotlib.pyplot as plt
import pprint
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import cohen_kappa_score, make_scorer
from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split
import scipy as sp
from sklearn.model_selection import train_test_split, GridSearchCV
from functools import partial
from collections import Counter
import random
import math
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
def metric(y1,y2):
return cohen_kappa_score(y1, y2, weights = 'quadratic')
 scorer = make_scorer(metric )
quadratic_weighted_kappa(target, train_predictions )<compute_test_metric>
def split_score(model, x_train, y_train, x_test, n=10):
y_pre = np.zeros(x_train.shape[0])
 oof_test = np.zeros(( x_test.shape[0], n))
kfold = StratifiedKFold(n_splits=n, random_state=4)
 aaa = 0
for train_index, test_index in kfold.split(x_train, y_train):
model.fit(x_train.iloc[train_index], y_train.iloc[train_index])
 y_pre[test_index] = model.predict(x_train.iloc[test_index])
 test_pred = model.predict(x_test)
 oof_test[:, aaa] = test_pred
print(aaa)
 aaa += 1
print("{}折后的Kappa加权得分为:带补充".format(n))
return y_pre, oof_test
def fix_y(y, coef):
y_fix = np.copy(y)
 for i, pred in enumerate(y_fix):
if pred < coef[0]:
y_fix[i] = 0
elif pred >= coef[0] and pred < coef[1]:
y_fix[i] = 1
elif pred >= coef[1] and pred < coef[2]:
y_fix[i] = 2
elif pred >= coef[2] and pred < coef[3]:
y_fix[i] = 3
else:
y_fix[i] = 4
return y_fix
def _kappa_loss(y, y_true, coef):
y_fix = np.copy(y)
 for i, pred in enumerate(y_fix):
if pred < coef[0]:
y_fix[i] = 0
elif pred >= coef[0] and pred < coef[1]:
y_fix[i] = 1
elif pred >= coef[1] and pred < coef[2]:
y_fix[i] = 2
elif pred >= coef[2] and pred < coef[3]:
y_fix[i] = 3
else:
y_fix[i] = 4
loss = metric(y_fix, y_true)
 return -loss
def search_coef(x1, x2):
loss_partial = partial(_kappa_loss, x1, x2)
 initial_coef = [1.55, 2.05, 2.5, 3]
coef = sp.optimize.basinhopping(loss_partial, initial_coef, niter=500, T=1,
stepsize=0.2, minimizer_kwargs={"method": 'nelder-mead'},
take_step=None, accept_test=None, callback=None,
interval=100, disp=True, niter_success=10, seed=None)
 return coef
rmse(target, [r[0] for r in results['train']] )<prepare_output>
df_train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 df_test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 train = df_train.copy()
test = df_test.copy()
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
def extract_sentiment_feature(i, x):
feature_sentiment = pd.DataFrame()
if x == 'train':
set_file = 'train'
else:
set_file = 'test'
file_name = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(set_file,i)
 try:
with open(file_name, 'r', encoding='utf-8')as f:
sentiment_file = json.load(f)
 token = [x['name'] for x in sentiment_file['entities']]
token = ' '.join(token)
 sentences_sentiment = [x['sentiment'] for x in sentiment_file['sentences']]
sentences_sentiment = pd.DataFrame.from_dict(
 sentences_sentiment, orient='columns')
 docementSentiment_magnitude = sentiment_file['documentSentiment']['magnitude']
documentSentiment_score = sentiment_file['documentSentiment']['score']
new = pd.DataFrame(
 {'PetID' : i,
'magnitude_sum' : sentences_sentiment['magnitude'].sum(axis=0),
'score_sum' : sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': sentences_sentiment['magnitude'].mean(axis=0),
'score_mean' : sentences_sentiment['score'].mean(axis=0),
'magnitude_var' : sentences_sentiment['magnitude'].var(axis=0),
'score_var' : sentences_sentiment['score'].var(axis=0),
'document_magnitude' : [docementSentiment_magnitude],
'document_score' : [documentSentiment_score]})
 feature_sentiment = feature_sentiment.append(new)
 except:
print('{}没找到'.format(file_name))
for each in feature_sentiment.columns:
if each not in ['PetID','token']:
feature_sentiment[each] = feature_sentiment[each].astype(float)
 return feature_sentiment
train_feature_sentiment = Parallel(n_jobs=8, verbose=1 )(
 delayed(extract_sentiment_feature )(i, 'train')for i in train.PetID)
 train_feature_sentiment = [x for x in train_feature_sentiment]
train_feature_sentiment = pd.concat(train_feature_sentiment, ignore_index=True, sort=False)
 test_feature_sentiment = Parallel(n_jobs=8, verbose=1 )(
 delayed(extract_sentiment_feature )(i, 'test')for i in test.PetID)
 test_feature_sentiment = [x for x in test_feature_sentiment]
test_feature_sentiment = pd.concat(test_feature_sentiment, ignore_index=True, sort=False )
submission.to_csv('submission.csv', index=False )<save_to_csv>
picture_metadata = pd.read_csv(r'.. /input/32111111/picture.csv')
 picture_size = pd.read_csv(r'.. /input/32111111/picture_size.csv')
 img_features = pd.read_csv(r'.. /input/32111111/img_features.csv')
 x_train = df_train.merge(train_feature_sentiment, how='left', on='PetID')
 x_train = x_train.merge(picture_metadata, how='left', on='PetID')
 x_train = x_train.merge(picture_size, how='left', on='PetID')
 x_train = x_train.merge(img_features, how='left', on='PetID')
 y_train = x_train['AdoptionSpeed']
x_train.drop(['AdoptionSpeed'], axis=1, inplace=True)
 x_test = df_test.merge(test_feature_sentiment, how='left', on='PetID')
 x_test = x_test.merge(picture_metadata, how='left', on='PetID')
 x_test = x_test.merge(picture_size, how='left', on='PetID')
 x_test = x_test.merge(img_features, how='left', on='PetID')
submission.to_csv('submission.csv', index=False )<import_modules>
col_text = ['Description']
x = x_train.append(x_test ).reset_index()
x = x[['Description', 'PetID']]
n_components = 50
x[col_text] = x[col_text].fillna('MISSING')
 text_features = []
for i in ['Description']:
svd_ = TruncatedSVD(n_components=n_components)
 tfv = CountVectorizer(min_df=3,
max_df=0.9,
stop_words = 'english')
 tfidf_col = tfv.fit_transform(x.loc[:, i])
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
text_features.append(svd_col)
 x.drop(i, axis=1, inplace=True)
 text_features = pd.concat(text_features, axis=1)
 x = pd.concat([x, text_features], axis=1)
 x_train = x_train.merge(x, how='left', on='PetID')
 x_test = x_test.merge(x, how='left', on='PetID')
import pandas as pd
import numpy as np
from sklearn.metrics import cohen_kappa_score,confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestRegressor
from functools import partial
import scipy as sp<load_from_csv>
x = x_train.append(x_test)
 x = x[['PetID', 'Description']]
x['Description'] = x['Description'].fillna("Missing")
 x['Description'] = x['Description'].apply(lambda x:TextBlob(x))
x['polarity'] = x['Description'].apply(lambda x:x.sentiment[0])
 x['subjectivity'] = x['Description'].apply(lambda x:x.sentiment[1])
 bin=[-2,0,0.3,2]
x['polarity'] = pd.cut(x['polarity'], bins=bin, labels=range(3))
x['polarity'] = x['polarity'].astype(np.int32)
 x_train = x_train.merge(x[['PetID', 'polarity']], how='left', on='PetID')
 x_test = x_test.merge(x[['PetID', 'polarity']], how='left', on='PetID' )
df = pd.read_csv('.. /input/train/train.csv' )<prepare_x_and_y>
x_train['IsFree'] = x_train['Fee'].apply(lambda x:True if x == 0 else False)
 x_test['IsFree'] = x_test['Fee'].apply(lambda x:True if x == 0 else False)
 x_train['Year'] = x_train['Age'].apply(lambda x:math.floor(x/12))
x_test['Year'] = x_test['Age'].apply(lambda x:math.floor(x/12))
x = x_train.append(x_test)
 x['Age_qcut'] = pd.qcut(x['Age'], 5, duplicates='drop')
 x['Age_qcut'] = pd.factorize(x['Age_qcut'])[0]
x_train = x_train.merge(x[['PetID','Age_qcut']], how='left', on='PetID')
 x_test = x_test.merge(x[['PetID','Age_qcut']], how='left', on='PetID')
 x = x_train.append(x_test ).reset_index()
Breed1_count = x.groupby('Breed1' ).size().to_frame('Breed1_count' ).reset_index()
x_train = x_train.merge(Breed1_count, how='left', on='Breed1')
 x_test = x_test.merge(Breed1_count, how='left', on='Breed1')
 a = x['Breed1'].value_counts().sort_values(ascending = False ).cumsum() /len(x)
 rare1_index = a[a > 0.85].index.tolist()
x_train['IsRare1'] = x_train['Breed1'].isin(rare1_index ).apply(lambda x:True if x == True else False)
 x_test['IsRare1'] = x_test['Breed1'].isin(rare1_index ).apply(lambda x:True if x == True else False)
 rare2_index = a[a > 0.72].index.tolist()
x_train['IsRare2'] = x_train['Breed1'].isin(rare2_index ).apply(lambda x:True if x == True else False)
 x_test['IsRare2'] = x_test['Breed1'].isin(rare2_index ).apply(lambda x:True if x == True else False)
 x_train['Is_COMMON'] = x_train['Breed1'].apply(lambda x:True if(x == 265 or x == 307 or x == 266)else False)
 x_test['Is_COMMON'] = x_test['Breed1'].apply(lambda x:True if(x == 265 or x == 307 or x == 266)else False)
 bin=[-0.5,0.5,1.5,4.5,1000]
x_train['Photo_cut'] = pd.cut(x_train['PhotoAmt'], bins=bin, labels=range(4)).astype(np.int32)
 x_test['Photo_cut'] = pd.cut(x_test['PhotoAmt'], bins=bin, labels=range(4)).astype(np.int32)
 x_train['Is_rare_color1'] = x_train['Color1'].apply(lambda x:True if x==5 or x==6 or x==7 else False)
 x_test['Is_rare_color1'] = x_test['Color1'].apply(lambda x:True if x==5 or x==6 or x==7 else False)
 x_train['Is_rare_color2'] = x_train['Color2'].apply(lambda x:True if x==6 else False)
 x_test['Is_rare_color2'] = x_test['Color2'].apply(lambda x:True if x==6 else False)
 x_train['Is_less_than_2month']= x_train['Age'].apply(lambda x:True if x<3 else False)
 x_test['Is_less_than_2month'] = x_test['Age'].apply(lambda x:True if x<3 else False)
dfDropped = df.drop(['Name','RescuerID','Description','PetID','AdoptionSpeed'],axis=1)
 X = dfDropped.values
y = df["AdoptionSpeed"]<compute_test_metric>
df = df_train.append(df_test)
 data_rescuer = df.groupby(['RescuerID'])['PetID'].size().reset_index()
data_rescuer.columns = ['RescuerID', 'RescuerID_count']
x_train = x_train.merge(data_rescuer, how='left', on='RescuerID')
 x_test = x_test.merge(data_rescuer, how='left', on='RescuerID')
 x = x_train.append(x_test)
 x['RescuerID_count_cut'] = pd.qcut(x['RescuerID_count'], 5, labels=range(4), duplicates='drop' ).astype(np.int32)
 x_train = x_train.merge(x[['PetID', 'RescuerID_count_cut']], how='left', on='PetID')
 x_test = x_test.merge(x [['PetID', 'RescuerID_count_cut']], how='left', on='PetID')
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p,weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<train_model>
x_train['HasSecondBreed'] = x_train['Breed2'].map(lambda x:True if x != 0 else False)
 x_test['HasSecondBreed'] = x_test['Breed2'].map(lambda x:True if x != 0 else False)
 train_breed_main = x_train[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = x_train[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 x_train = pd.concat(
 [x_train, train_breed_main, train_breed_second], axis=1)
 test_breed_main = x_test[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = x_test[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 x_test = pd.concat(
 [x_test, test_breed_main, test_breed_second], axis=1)
 print(x_train.shape, x_test.shape)
 categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
x_train['True_Pure'] = False
x_train.loc[(x_train['main_breed_BreedName'] != 'Mixed Breed')& 
(( x_train['main_breed_BreedName'] == x_train['second_breed_BreedName'])| 
(x_train['second_breed_BreedName'].isnull())) ,'True_Pure'] = True
x_test['True_Pure'] = False
x_test.loc[(x_test['main_breed_BreedName'] != 'Mixed Breed')& 
(( x_test['main_breed_BreedName'] == x_test['second_breed_BreedName'])| 
(x_test['second_breed_BreedName'].isnull())) ,'True_Pure'] = True
x_train['Is_Pure_Dog'] =(x_train['True_Pure'] == True)&(x_train['Type'] == 1)
 x_test['Is_Pure_Dog'] =(x_test['True_Pure'] == True)&(x_test['Type'] == 1)
 x_train.drop(['main_breed_BreedName', 'second_breed_BreedName', 'main_breed_Type', 'second_breed_Type'], axis=1, inplace=True)
 x_test.drop(['main_breed_BreedName', 'second_breed_BreedName', 'main_breed_Type', 'second_breed_Type'], axis=1, inplace=True )
rfr = RandomForestRegressor(max_depth=10, n_estimators=300)
 rfr.fit(X,y )<train_model>
drop_columns = ['Name', 'RescuerID', 'Description', 'PetID', 'token', 'annots_top_desc']
drop_columns = ['Name', 'RescuerID', 'Description', 'PetID']
x_train.drop(drop_columns, axis=1, inplace=True)
 x_test.drop(drop_columns, axis=1, inplace=True)
 x_train = x_train.fillna(0)
 x_test = x_test.fillna(0)
optr = OptimizedRounder()
optr.fit(rfr.predict(X),y )<load_from_csv>
c = ['Breed1', 'Breed2', 'Gender', 'Color1', 'Color2', 'Color3',
'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'Quantity',
'State', 'IsFree', 'Year', 'Age_qcut', 'IsRare1', 'IsRare2',
'Is_COMMON', 'Photo_cut', 'Is_rare_color1', 'Is_rare_color2', 'Is_less_than_2month',
'RescuerID_count_cut', 'HasSecondBreed', 'True_Pure', 'Is_Pure_Dog']
for each in c:
x_train[each] = x_train[each].astype('category')
 x_test[each] = x_test[each].astype('category' )
df_t = pd.read_csv('.. /input/test/test.csv')
 X_t = df_t.drop(['Name','RescuerID','Description','PetID'],axis=1 ).values<save_to_csv>
model_lgb = LGBMRegressor(
 learning_rate = 0.01,
n_estimators = 2000,
max_depth = 4,
num_leaves = 12 ,
subsample = 0.8,
colsample_bytree = 0.8,
n_jobs = -1,
random_state = 44,
objective = 'regression',
eval_metric = 'scorer',
min_child_samples = 15 
)
submission = pd.DataFrame(dict(PetID=df_t['PetID'], AdoptionSpeed=optr.predict(rfr.predict(X_t), optr.coefficients() ).astype(int)))
 submission.to_csv('submission.csv', index=False )<set_options>
y_lgb, y_test_pre = split_score(model_lgb, x_train, y_train, x_test)
 y_test_pre = y_test_pre.mean(axis=1)
 coe = search_coef(y_lgb, y_train)
 best_lgb_coe = coe['x']
print('lgb的最佳系数为{}'.format(best_lgb_coe))
model_lgb.fit(x_train, y_train)
 result_lgb_fix = fix_y(y_test_pre, best_lgb_coe)
 print('lgb后的分布:',Counter(result_lgb_fix))
<load_from_csv><EOS> 
submission_lgb = pd.DataFrame({'PetID': df_test['PetID'].values, 'AdoptionSpeed': result_lgb_fix.astype(np.int32)})
 submission_lgb.to_csv('submission.csv', index=False )