Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<split>
%matplotlib inline
split_char = '/'
X_train_part, X_valid, y_train_part, y_valid, train_part_types, valid_types = split(X_train, y_train, train_types,
random_state=42, p_value=0.0 )<split>
breeds = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 colors = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 states = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv' )
X_train_small, _, y_train_small, _, train_small_types, _ = split(X_train, y_train, train_types, test_size=0.90 )<categorify>
train['Age'] = train['Age'].fillna(method='pad')
 train.loc[train['Age'] > 20, 'Age'] = train['Age'].mean()
feat_cols_selected_indices = [idx for idx, name in enumerate(feat_cols)if name in feat_cols_selected]
X_train = X_train[:, feat_cols_selected_indices]
X_train_part = X_train_part[:, feat_cols_selected_indices]
X_valid = X_valid[:, feat_cols_selected_indices]
X_train_small = X_train_small[:, feat_cols_selected_indices]
X_test = X_test[:, feat_cols_selected_indices]<import_modules>
breed_dict = dict(zip(breeds['BreedID'], breeds['BreedName']))
from sklearn.ensemble import ExtraTreesRegressor<choose_model_class>
animal_dict = {1: 'Dog', 2: 'Cat'}
reg = ExtraTreesRegressor(n_estimators=10, n_jobs=-1, random_state=42, verbose=2 )<train_model>
encodedColor1 = pd.get_dummies(train['Color1'], prefix="color")
 train = pd.concat([train, encodedColor1], axis='columns')
 encodedColor2 = pd.get_dummies(test['Color1'], prefix="color")
 test = pd.concat([test, encodedColor2], axis='columns' )
%%time
reg.fit(X_train_part, y_train_part )<predict_on_test>
img_size = 256
batch_size = 16
y_pred = reg.predict(X_valid )<compute_test_metric>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
def metric(y, y_hat, types, type_filter=None, verbose=True):
res = 0
uniq_types = np.unique(types)
 if verbose:
print('typ| cnt(%)| sum | log')
 cnt_total = types.shape[0]
for typ in uniq_types:
if type_filter is not None and type_filter != typ:
continue
idx = np.where(types == typ)[0]
cnt = idx.shape[0]
res_typ = np.sum(np.abs(y[idx] - y_hat[idx])) / cnt
assert cnt == y[idx].shape[0] == y_hat[idx].shape[0], 'inconsistent idx'
if verbose:
print(f'{typ:3}|{cnt:8}({1+cnt*100//cnt_total:02d})|{res_typ:12.9}|{np.log(res_typ)}')
 res += np.log(res_typ)
 uniq_types_cnt = uniq_types.shape[0] if type_filter is None else 1
res /= uniq_types_cnt
if verbose:
print(f'Result: {res} for {uniq_types_cnt} types')
 return res<compute_test_metric>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
metric(y_valid, y_pred, valid_types )<import_modules>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
from skopt.space import Real, Integer, Categorical<define_search_space>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
space = [
Integer(10, 100, name='n_estimators'),
Real(0.1, 1.0, prior='uniform', name='max_features'),
]<define_search_space>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
n_estimators_by_type = {
0: 32, 1: 160, 2: 28, 3: 100, 4: 160, 5: 28, 6: 32, 7: 160
}<choose_model_class>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = ['pic_'+str(i)for i in range(train_feats.shape[1])]
def get_estimator(estimator_model='rf', n_estimators=10, verbose=False):
if estimator_model == 'rf':
cls = RandomForestRegressor
elif estimator_model == 'et':
cls = ExtraTreesRegressor
elif estimator_model == 'lgbm':
cls = LGBMRegressor
else:
return None
print('Selected estimator class:', cls)
 return cls(n_estimators=n_estimators, n_jobs=-1, random_state=42, verbose=verbose )<train_model>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
def skopt_hyper(space, n_estimators_by_type,
X_train, y_train, train_types,
X_valid, y_valid, valid_types,
estimator_model='rf', n_calls=30,
random_state=42, verbose=True):
reg = get_estimator(estimator_model, verbose=0)
 if reg is None:
return None
res_opts = {}
for typ in tqdm_notebook(np.unique(train_types)) :
@use_named_args(space)
 def objective(**params):
reg.set_params(**params)
 mask = train_types == typ
reg.fit(X_train[mask], y_train[mask])
 y_valid_pred = reg.predict(X_valid)
 score = metric(y_valid, y_valid_pred, valid_types, typ, verbose=verbose)
 return score
if n_estimators_by_type is not None:
space[0] = Integer(10, n_estimators_by_type[typ], name='n_estimators')
 if verbose:
print(f'--- Type {typ}, space {space} ---')
 res_opt = gp_minimize(objective, space, n_calls=n_calls,
verbose=True, random_state=random_state)
 res_opts[typ] = res_opt
if verbose:
print(f'--- optimal params: {res_opt.x} ---')
 for typ in np.unique(train_types):
if verbose:
print('Type', typ)
 res_opt = res_opts[typ]
plot_convergence(res_opt)
 plot_evaluations(res_opt)
 plot_objective(res_opt)
 plt.show()
return {k: v.x for k, v in res_opts.items() }<train_on_grid>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
%time
res_opts = {
0: [32, 0.9356368713812235],
1: [160, 0.9226985331078089],
2: [28, 0.978079768528465],
3: [100, 1.0],
4: [160, 1.0],
5: [28, 0.9688476690205474],
6: [32, 1.0],
7: [160, 0.9976351777608269]
}<find_best_model_class>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = ['pic_'+str(i)for i in range(test_feats.shape[1])]
def train_and_predict_sep(X_train, y_train, X_train_part, y_train_part, X_valid, y_valid,
train_types, train_part_types, valid_types,
X_test=None, test_types=None,
estimator_model='rf', cnt_estimators=None, res_opts=None, params=None):
assert X_train.shape[0] == y_train.shape[0] == train_types.shape[0], 'Inconsistent train - X and y'
assert X_train_part.shape[0] == y_train_part.shape[0] == train_part_types.shape[0], 'Inconsistent train_part - X and y'
assert X_valid.shape[0] == y_valid.shape[0] == valid_types.shape[0], 'Inconsistent valid - X and y'
y_pred_valid = np.empty(X_valid.shape[0])
 y_pred_test = None
if X_test is not None and test_types is not None:
assert X_test.shape[0] == test_types.shape[0], 'Inconsistent train - X and y'
y_pred_test = np.empty(X_test.shape[0])
 typ_cnt = np.unique(train_types ).shape[0]
if cnt_estimators is None:
cnt_estimators = [10] * typ_cnt
assert len(cnt_estimators)== typ_cnt, 'Inconsistent estimators count'
for n_estimators, typ in tqdm_notebook(zip(cnt_estimators, range(typ_cnt)) , total=len(cnt_estimators)) :
print('+++++ Fitting model for typ', typ)
 mask = train_part_types == typ
reg = get_estimator(estimator_model, cnt_estimators, verbose=2)
 if reg is None:
return None
if params is not None:
reg.set_params(**params[typ])
 elif res_opts is not None:
res_opt = res_opts[typ]
reg_params = {
'n_estimators': res_opt[0],
'max_features': res_opt[1],
}
reg.set_params(**reg_params)
 print(reg)
 reg.fit(X_train_part[mask], y_train_part[mask])
 print('+++++ Predicting with model for typ', typ)
 mask = valid_types == typ
y_pred_valid[mask] = reg.predict(X_valid[mask])
 metric(y_valid, y_pred_valid, valid_types, typ)
 if X_test is not None and test_types is not None:
print('+++++ Fitting model for typ', typ)
 mask = train_types == typ
reg.fit(X_train[mask], y_train[mask])
 print('+++++ Predicting with model for typ', typ)
 mask = test_types == typ
y_pred_test[mask] = reg.predict(X_test[mask])
 print('+++++ Overall validation report')
 metric(y_valid, y_pred_valid, valid_types)
 return y_pred_test, y_pred_valid<define_variables>
test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
cnt_estimators = None<predict_on_test>
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train = pd.merge(train, train_feats, how='left', on='PetID')
 test = pd.merge(test, test_feats, how='left', on='PetID')
%%time
y_pred, _ = train_and_predict_sep(X_train, y_train, X_train_part, y_train_part, X_valid, y_valid,
train_types, train_part_types, valid_types,
X_test=X_test, test_types=test_types,
estimator_model='et', cnt_estimators=cnt_estimators, res_opts=res_opts )<load_from_csv>
import glob
import json
df_subm = pd.read_csv('.. /input/champs-scalar-coupling/sample_submission.csv', index_col='id' )<prepare_output>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
df_subm[target] = y_pred<load_from_csv>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
fn_csv = f'./{EXP_NUMBER}.subm.csv'<save_to_csv>
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
df_subm.to_csv(fn_csv )<set_options>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
warnings.filterwarnings("ignore")
 warnings.filterwarnings(action="ignore",category=DeprecationWarning)
 warnings.filterwarnings(action="ignore",category=FutureWarning)
<define_variables>
train_breed_main = train_proc[['Breed1']].merge(
 breeds, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 breeds, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 breeds, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 breeds, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
DATA_PATH = '.. /input/champs-scalar-coupling'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<load_from_csv>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )<load_from_csv>
X_temp = X.copy()
to_drop_columns = ['PetID', 'Name', 'RescuerID', 'Description', 'metadata_annots_top_desc', 'sentiment_entities', 'main_breed_BreedName', 'second_breed_BreedName']
X_temp = X_temp.drop(to_drop_columns, axis=1 )
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv' )<load_from_csv>
X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )<data_type_conversions>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )<data_type_conversions>
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures<merge>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df<merge>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
<merge><EOS> 
test_predictions = optR.predict(oof_test.mean(axis=1), coefficients ).astype(np.int8)
 submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()