Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<feature_engineering>
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic')
 def warn(*args, **kwargs):
pass
warnings.warn = warn
print(os.listdir(".. /input"))
train_df['Mixed_Breed'] = train_df.apply(lambda x: 0 if x.Breed2==0 and x.Breed1!=307 else 1, axis=1)
 train_df['Num_Color'] = train_df.apply(lambda x: 3-sum([y==0 for y in [x.Color1, x.Color2, x.Color3]]), axis=1)
 train_df['Description'].fillna("", inplace=True)
 train_df['Description_Length'] = train_df.Description.map(len )<feature_engineering>
data_path = ".. /input/petfinder-adoption-prediction/"
train_df = pd.read_csv(data_path + "train/train.csv")
 test_df = pd.read_csv(data_path + "test/test.csv")
 train_img = pd.read_csv(".. /input/petfindermy-dense121-extracted-image-features/train_img_features.csv")
 test_img = pd.read_csv(".. /input/petfindermy-dense121-extracted-image-features/test_img_features.csv")
 train_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 test_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 train_df = pd.concat([train_df, train_img], axis=1)
 test_df = pd.concat([test_df, test_img], axis=1)
 df = pd.concat([train_df, test_df], axis=0)
 df.head(2 )
test_df['Mixed_Breed'] = test_df.apply(lambda x: 0 if x.Breed2==0 and x.Breed1!=307 else 1, axis=1)
 test_df['Num_Color'] = test_df.apply(lambda x: 3-sum([y==0 for y in [x.Color1, x.Color2, x.Color3]]), axis=1)
 test_df['Description'].fillna("", inplace=True)
 test_df['Description_Length'] = test_df.Description.map(len )<create_dataframe>
train_metadata_files = sorted(glob.glob(data_path + 'train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob(data_path + 'train_sentiment/*.json'))
train_metadata_df = pd.DataFrame(train_metadata_files, columns=["filename"])
 train_sentiment_df = pd.DataFrame(train_sentiment_files, columns=["filename"])
 test_metadata_files = sorted(glob.glob(data_path + 'test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob(data_path + 'test_sentiment/*.json'))
test_metadata_df = pd.DataFrame(test_metadata_files, columns=["filename"])
 test_sentiment_df = pd.DataFrame(test_sentiment_files, columns=["filename"])
 sentiment_df = pd.concat([train_sentiment_df, test_sentiment_df])
 metadata_df = pd.concat([train_metadata_df, test_metadata_df])
 del train_sentiment_df; del test_sentiment_df
del train_metadata_df; del test_metadata_df
def assign_pet_id(df):
df["PetID"] = df["filename"].apply(lambda x: x.split("/")[-1].split("-")[0].split(".")[0])
 assign_pet_id(metadata_df)
 assign_pet_id(sentiment_df )
def get_desc_anly(type, recalc):
if recalc == 1:
if type == "train":
path = ".. /input/train_sentiment/"
elif type == "test":
path = ".. /input/test_sentiment/"
print("Getting description sentiment analysis for",type+"_sentiment.csv")
 files = [f for f in os.listdir(path)if(f.endswith('.json')& os.path.isfile(path+f)) ]
df = pd.DataFrame(columns=["PetID", "DescScore", "DescMagnitude"])
 i = 0
for f in files:
with open(path+f, encoding="utf8")as json_data:
data = json.load(json_data)
 df.loc[i]= [f[:-5],data["documentSentiment"]["score"],data["documentSentiment"]["magnitude"]]
i = i+1
df.to_csv(type+"_sentiment.csv", index=False)
 elif recalc == 0:
df = pd.read_csv(type+"_sentiment.csv")
 return df
train_snt = get_desc_anly("train", 1)
 test_snt = get_desc_anly("test", 1)
 train_df = train_df.set_index("PetID" ).join(train_snt.set_index("PetID")).reset_index()
test_df = test_df.set_index("PetID" ).join(test_snt.set_index("PetID")).reset_index()
train_df["DescScore"].fillna(0, inplace=True)
 train_df["DescMagnitude"].fillna(0, inplace=True)
 train_df["Name"].fillna("none", inplace=True)
 test_df["DescScore"].fillna(0, inplace=True)
 test_df["DescMagnitude"].fillna(0, inplace=True)
 test_df["Name"].fillna("", inplace=True)
 train_df["NameLength"] = train_df["Name"].str.len()
test_df["NameLength"] = test_df["Name"].str.len() 
<prepare_x_and_y>
def extract_sentiments(row):
with open(row["filename"], 'r')as f:
file = json.load(f)
 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = ' '.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 for key, value in file_sentiment.items() :
row["sentiment_"+key] = value
return row
sentiment_df = sentiment_df.apply(extract_sentiments, axis=1)
 sentiment_df.head()
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train_df.loc[:, 'vertex_x'] = vertex_xs
train_df.loc[:, 'vertex_y'] = vertex_ys
train_df.loc[:, 'bounding_confidence'] = bounding_confidences
train_df.loc[:, 'bounding_importance'] = bounding_importance_fracs
train_df.loc[:, 'dominant_blue'] = dominant_blues
train_df.loc[:, 'dominant_green'] = dominant_greens
train_df.loc[:, 'dominant_red'] = dominant_reds
train_df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train_df.loc[:, 'dominant_score'] = dominant_scores
train_df.loc[:, 'label_description'] = label_descriptions
train_df.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test_df.loc[:, 'vertex_x'] = vertex_xs
test_df.loc[:, 'vertex_y'] = vertex_ys
test_df.loc[:, 'bounding_confidence'] = bounding_confidences
test_df.loc[:, 'bounding_importance'] = bounding_importance_fracs
test_df.loc[:, 'dominant_blue'] = dominant_blues
test_df.loc[:, 'dominant_green'] = dominant_greens
test_df.loc[:, 'dominant_red'] = dominant_reds
test_df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test_df.loc[:, 'dominant_score'] = dominant_scores
test_df.loc[:, 'label_description'] = label_descriptions
test_df.loc[:, 'label_score'] = label_scores<drop_column>
def extract_metadata(row):
with open(row["filename"], 'r')as f:
file = json.load(f)
 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else: file_crop_importance = np.nan
metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
}
for key, value in metadata.items() :
row["metadata_"+key] = value
return row
metadata_df = metadata_df.apply(extract_metadata, axis=1)
 metadata_df.head()
train_df_num = train_df.drop(columns = train_df.dtypes[train_df.dtypes=='object'].index )<drop_column>
aggregates = ["sum", "mean"]
metadata_agg = metadata_df.drop(["filename"], axis=1 ).groupby(["PetID"] ).agg(aggregates)
 metadata_agg.columns = pd.Index([f"metadata_{c[0]}_{c[1].upper() }" for c in metadata_agg.columns.tolist() ])
 metadata_agg = metadata_agg.reset_index()
sentiment_agg = sentiment_df.drop(["filename"], axis=1 ).groupby(["PetID"] ).agg(aggregates)
 sentiment_agg.columns = pd.Index([f"sentiment_{c[0]}_{c[1].upper() }" for c in sentiment_agg.columns.tolist() ])
 sentiment_agg = sentiment_agg.reset_index()
test_df_num = test_df.drop(columns = test_df.dtypes[test_df.dtypes=='object'].index )<import_modules>
df = df.merge(metadata_agg, how="left", on="PetID")
 df = df.merge(sentiment_agg, how="left", on="PetID")
 df.head()
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score, cohen_kappa_score, make_scorer
from sklearn.model_selection import StratifiedKFold
from sklearn.utils import shuffle<prepare_x_and_y>
word_vec_size = 300
max_words = 100
max_word_features = 25000
def transform_text(text, tokenizer):
tokenizer.fit_on_texts(text)
 text_emb = tokenizer.texts_to_sequences(text)
 text_emb = sequence.pad_sequences(text_emb, maxlen=max_words)
 return text_emb
desc_tokenizer = text.Tokenizer(num_words=max_word_features)
 desc_embs = transform_text(df["Description"].astype(str), desc_tokenizer )
x = train_df_num.drop(columns = 'AdoptionSpeed')
 y = train_df_num.AdoptionSpeed<prepare_x_and_y>
text_mode = "word2vec"
if text_mode == "fasttext":
embedding_file = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(embedding_file))
word_index = desc_tokenizer.word_index
print('Word index len:', len(word_index))
vocabulary_size = min(max_word_features, len(word_index)) + 1
text_embs = np.zeros(( vocabulary_size, word_vec_size))
for word, i in word_index.items() :
if i >= max_word_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: text_embs[i] = embedding_vector
del(embeddings_index)
 elif text_mode == "word2vec":
embedding_file = ".. /input/word2vec-google/GoogleNews-vectors-negative300.bin"
print("Loading word vectors...")
 word_vectors = KeyedVectors.load_word2vec_format(embedding_file, binary=True)
 print("Matching word vectors...")
 EMBEDDING_DIM=300
word_index = desc_tokenizer.word_index
vocabulary_size=min(len(word_index)+1,max_word_features)
 text_embs = np.zeros(( vocabulary_size, EMBEDDING_DIM))
for word, i in word_index.items() :
if i>=max_word_features:
continue
try:
embedding_vector = word_vectors[word]
text_embs[i] = embedding_vector
except KeyError:
text_embs[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)
 del(word_vectors )
x_tst = test_df_num<split>
cat_vars = ["Type", "Breed1", "Breed2", "Color1", "Color2", "Color3", "Gender", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "Health", "State"]
cont_vars = ["Fee", "PhotoAmt", "VideoAmt", "Age", "Quantity", "NameLength"]
def preproc(df):
global cont_vars
df["NameLength"] = df["Name"].astype(str ).apply(len)
 for var in cat_vars:
df[var] = LabelEncoder().fit_transform(df[var])
 for var in cont_vars:
df[var] = MinMaxScaler().fit_transform(df[var].values.reshape(-1,1))
df[var].fillna(df[var].mean() , inplace=True)
 return df
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.1, random_state=0, shuffle=True )<define_variables>
df = preproc(df)
 train_df = df[:len(train_df)]
test_df = df[len(train_df):]
len(train_df), len(test_df)
 del df
train_df.head(2 )
ExtraTreesClassifier, AdaBoostClassifier
RANDOM_STATE = 0<choose_model_class>
def get_keras_data(df, description_embeds):
X = {var: df[var].values for var in cont_vars+cat_vars}
X["description"] = description_embeds
for i in range(256): X[f"img_{i}"] = df[f"img_{i}"]
return X
classifiers = [KNeighborsClassifier(n_jobs=-1),
LogisticRegression() ,
LogisticRegression(C=0.005, penalty='l1', class_weight=None, fit_intercept=False, max_iter=100, tol=0.01),
LogisticRegression(C=0.0001, penalty='l2', class_weight=None, fit_intercept=True, max_iter=150, tol=1e-06),
LogisticRegression(C=96, penalty='l1', class_weight='balanced', fit_intercept=True, max_iter=50, tol=0.0001),
LogisticRegression(C=54, penalty='l2', class_weight='balanced', fit_intercept=False, max_iter=450, tol=0.001),
GradientBoostingClassifier(n_estimators=50, learning_rate=0.005, max_depth=12, max_features=0.8, min_samples_leaf=2, subsample=0.2),
GradientBoostingClassifier(n_estimators=50, learning_rate=0.01, max_depth=5, max_features=0.6, min_samples_leaf=10, subsample=0.8),
GradientBoostingClassifier(n_estimators=200, learning_rate=0.001, max_depth=90, max_features=0.5, min_samples_leaf=20, subsample=0.2),
GradientBoostingClassifier(n_estimators=50, learning_rate=0.01, max_depth=20, max_features=0.6, min_samples_leaf=24, subsample=0.7),
GradientBoostingClassifier(n_estimators=300, learning_rate=0.01, max_depth=90, max_features=0.5, min_samples_leaf=20, subsample=0.2),
RandomForestClassifier(n_jobs=-1),
RandomForestClassifier(n_estimators=225, bootstrap=True, max_depth=83, max_features='auto', min_samples_leaf=5, min_samples_split=5, n_jobs=-1),
RandomForestClassifier(criterion='gini', n_estimators=100, min_samples_split=12, min_samples_leaf=1, oob_score=True, n_jobs=-1),
ExtraTreesClassifier(n_jobs=-1),
ExtraTreesClassifier(n_estimators=200, bootstrap=False, max_depth=80, max_features='auto', min_samples_leaf=6, min_samples_split=5, n_jobs=-1),
AdaBoostClassifier() ,
AdaBoostClassifier(n_estimators=225, algorithm='SAMME.R', learning_rate=0.2),
DecisionTreeClassifier() ,
DecisionTreeClassifier(criterion='entropy', max_depth=110, max_features='auto', min_samples_leaf=6, min_samples_split=330)
 ]
classifiers_names = ['knn1',
'lr1', 'lr2', 'lr3', 'lr4', 'lr5',
'gb1', 'gb2', 'gb3', 'gb4', 'gb5',
'rf1', 'rf2', 'rf3',
'et1', 'et2',
'adb1', 'adb2',
'dt1','dt2',
]<compute_train_metric>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
classifiers_predictions = pd.DataFrame()
for name, classifier in zip(classifiers_names, classifiers):
classifier.fit(X_train, y_train)
 train_predictions = pd.Series(classifier.predict(X_train))
test_predictions = classifier.predict(X_test)
 classifiers_predictions[name] = test_predictions
print('{0}:({1} - {2})'.format(name,
cohen_kappa_score(y_train, train_predictions, weights='quadratic'),
cohen_kappa_score(y_test, test_predictions, weights='quadratic')) )<prepare_output>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
classifiers_predictions = classifiers_predictions[['knn1',
'lr1', 'lr2', 'lr3', 'lr4', 'lr5',
'gb1', 'gb2', 'gb3', 'gb4', 'gb5',
'rf1', 'rf2', 'rf3',
'et1', 'et2',
'adb1', 'adb2',
'dt1','dt2',
]]<choose_model_class>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class QWKEvaluation(Callback):
def __init__(self, train_data=() , validation_data=() , measure_train=False, interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.history = []
self.X_val, self.y_val = validation_data
self.X_train, self.y_train = train_data
self.measure_train = measure_train
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, batch_size=3000, verbose=0)
 y_pred = eval_predict(self.y_val, y_pred)
 val_score = quadratic_weighted_kappa(self.y_val, y_pred)
 if self.measure_train:
y_pred = self.model.predict(self.X_train, batch_size=3000, verbose=0)
 y_pred = eval_predict(self.y_train, y_pred)
 train_score = quadratic_weighted_kappa(self.y_train, y_pred)
 else: train_score = 0
print("QWK - epoch: %d - train_score: %.6f, val_score: %.6f
" %(epoch+1, train_score, val_score))
self.history.append(val_score)
 if val_score >= max(self.history): self.model.save('checkpoint.h5')
 def eval_predict(y=[], y_pred=[], coeffs=None, ret_coeffs=False):
optR = OptimizedRounder()
if not coeffs:
optR.fit(y_pred.reshape(-1,), y)
 coeffs = optR.coefficients()
if ret_coeffs: return optR.coefficients()
return optR.predict(y_pred, coeffs ).reshape(-1, )
classifiers = [
LogisticRegression(C=96, penalty='l1', class_weight='balanced', fit_intercept=True, max_iter=50, tol=0.0001),
GradientBoostingClassifier(n_estimators=50, learning_rate=0.005, max_depth=12, max_features=0.8, min_samples_leaf=2, subsample=0.2),
GradientBoostingClassifier(n_estimators=50, learning_rate=0.01, max_depth=5, max_features=0.6, min_samples_leaf=10, subsample=0.8),
GradientBoostingClassifier(n_estimators=200, learning_rate=0.001, max_depth=90, max_features=0.5, min_samples_leaf=20, subsample=0.2),
GradientBoostingClassifier(n_estimators=50, learning_rate=0.01, max_depth=20, max_features=0.6, min_samples_leaf=24, subsample=0.7),
RandomForestClassifier(n_estimators=225, bootstrap=True, max_depth=83, max_features='auto', min_samples_leaf=5, min_samples_split=5, n_jobs=-1),
RandomForestClassifier(criterion='gini', n_estimators=100, min_samples_split=12, min_samples_leaf=1, oob_score=True, n_jobs=-1),
ExtraTreesClassifier(n_estimators=200, bootstrap=False, max_depth=80, max_features='auto', min_samples_leaf=6, min_samples_split=5, n_jobs=-1),
AdaBoostClassifier() ,
AdaBoostClassifier(n_estimators=225, algorithm='SAMME.R', learning_rate=0.2),
DecisionTreeClassifier(criterion='entropy', max_depth=110, max_features='auto', min_samples_leaf=6, min_samples_split=330)
 ]
classifiers_names = ['lr4',
'gb1','gb2', 'gb3', 'gb4',
'rf2', 'rf3',
'et2',
'adb1','adb2',
'dt2',
]<split>
def rmse(y, y_pred):
return K.sqrt(K.mean(K.square(y-y_pred), axis=-1))
def get_model(emb_n=10, dout=.25, batch_size=1000):
inps = []
embs = []
nums = []
for var in cat_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 embs.append(( Embedding(train_df[var].max() +1, emb_n )(inp)))
 for var in cont_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 nums.append(( inp))
desc_inp = Input(shape=(max_words,), name="description")
 inps.append(desc_inp)
 emb_desc = Embedding(vocabulary_size, word_vec_size, weights=[text_embs] )(desc_inp)
 emb_desc = SpatialDropout1D (.4 )(emb_desc)
 emb_desc = Reshape(( max_words, word_vec_size, 1))(emb_desc)
 filter_sizes=[1,3]
convs = []
for filter_size in filter_sizes:
conv = Conv2D(32, kernel_size=(filter_size, word_vec_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool2D(pool_size=(max_words-filter_size+1, 1))(conv))
img_fts = []
for i in range(256):
inp = Input(shape=[1], name=f"img_{i}")
 inps.append(inp)
 img_fts.append(( inp))
img_fts = concatenate(img_fts)
 img_fts = BatchNormalization()(img_fts)
 img_fts = Dense(64, activation="relu", kernel_initializer="he_normal" )(img_fts)
 img_fts = Dropout (.2 )(img_fts)
 convs = concatenate(convs)
 convs = Flatten()(convs)
 convs = BatchNormalization()(convs)
 embs = Flatten()(concatenate(embs))
embs = Dropout(dout )(Dense(64, activation="relu", kernel_initializer="he_normal" )(embs))
nums = concatenate(nums)
 nums = Dense(32, activation="relu" )(nums)
 x = concatenate([embs, nums, convs, img_fts])
 x = BatchNormalization()(x)
 dense_n = [256, 64]
for n in dense_n:
x = BatchNormalization()(x)
 x = Dense(n, activation="relu", kernel_initializer="he_normal" )(x)
 x = BatchNormalization()(x)
 x = Dropout (.3 )(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inps, outputs=out)
 opt = Adam()
model.compile(optimizer=opt, loss=rmse,)
 return model
def simple_blending(basic_algorithms, meta_algorithm, X_train, X_test, y_train, test_df, part1_ratio=0.9, random_state=None):
tr = pd.DataFrame()
tst = pd.DataFrame()
y = pd.DataFrame()
X_train_part1, X_train_part2,\
y_train_part1, y_train_part2 = train_test_split(X_train, y_train, test_size=1-part1_ratio, random_state=random_state)
 for index, basic_algorithm in enumerate(basic_algorithms):
basic_algorithm.fit(X_train_part1, y_train_part1)
 part2_predictions = basic_algorithm.predict(X_train_part2)
 tr[index] = part2_predictions
test_predictions = basic_algorithm.predict(X_test)
 tst[index] = test_predictions
test_pred = basic_algorithm.predict(test_df)
 y[index] = test_pred
meta_algorithm.fit(tr, y_train_part2)
 return meta_algorithm.predict(tst), meta_algorithm.predict(y )<create_dataframe>
nfolds=5
folds = GroupKFold(n_splits=nfolds)
 avg_train_kappa = 0
avg_valid_kappa = 0
batch_size=1000
coeffs=None
X_test = get_keras_data(test_df, desc_embs[len(train_df):])
 submission_df = test_df[["PetID"]]
adoptions = np.zeros(( len(test_df),))
for train_idx, valid_idx in folds.split(train_df[cat_vars+cont_vars], train_df["AdoptionSpeed"], groups=train_df["RescuerID"]):
X_train = get_keras_data(train_df.iloc[train_idx], desc_embs[train_idx])
 X_valid = get_keras_data(train_df.iloc[valid_idx], desc_embs[valid_idx])
 y_train, y_valid = train_df["AdoptionSpeed"][train_idx].values, train_df["AdoptionSpeed"][valid_idx].values
model = get_model()
clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(train_df)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(train_data=(X_train, y_train),validation_data=(X_valid, y_valid),
measure_train=False, interval=1)
 history = model.fit(X_train, y_train, batch_size=batch_size, validation_data=(X_valid, y_valid),
epochs=5, callbacks=[clr_tri, qwk_eval])
 model.load_weights('checkpoint.h5')
 y_train_pred = eval_predict(y_train, model.predict(X_train, batch_size=1000))
y_valid_pred = eval_predict(y_valid, model.predict(X_valid, batch_size=1000))
avg_train_kappa += quadratic_weighted_kappa(y_train_pred, y_train)/nfolds
avg_valid_kappa += quadratic_weighted_kappa(y_valid_pred, y_valid)/nfolds
coeffs = eval_predict(y_valid, model.predict(X_valid, batch_size=1000), ret_coeffs=True)
 adoptions += model.predict(X_test, batch_size=batch_size ).reshape(-1,)/nfolds
print("
avg train kappa:", avg_train_kappa,)
 print("
avg valid kappa:", avg_valid_kappa,)
<data_type_conversions><EOS> 
coeffs[0] = 1.645
coeffs[1] = 2.115
coeffs[3] = 2.84
submission_df["AdoptionSpeed"] = eval_predict(y_pred=adoptions, coeffs=list(coeffs)).astype(int)
 submission_df.to_csv("submission.csv", index=False)
 submission_df.head()