Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<create_dataframe>
palette = sns.color_palette("Paired")
 sns.set()
sns.set_palette(palette)
 split_char = '/'
print(os.listdir("./"))
test_id = pd.DataFrame(test_id )<merge>
trainPath = '.. /input/petfinder-adoption-prediction/train/train.csv'
testPath = '.. /input/petfinder-adoption-prediction/test/test.csv'
trainSentimentPath = '.. /input/petfinder-adoption-prediction/train_sentiment/'
testSentimentPath = '.. /input/petfinder-adoption-prediction/test_sentiment/'
trainMetadataPath = '.. /input/petfinder-adoption-prediction/train_metadata/'
testMetadataPath = '.. /input/petfinder-adoption-prediction/test_metadata/'
trainImagePath = '.. /input/petfinder-adoption-prediction/train_images/'
testImagePath = '.. /input/petfinder-adoption-prediction/test_images/'
breedPath = '.. /input/petfinder-adoption-prediction/breed_labels.csv'
colorPath = '.. /input/petfinder-adoption-prediction/color_labels.csv'
statePath = '.. /input/petfinder-adoption-prediction/state_labels.csv'
trainPrecomputedPath = '.. /input/precomputedfeaturespetfinder/train_precomputed.csv'
testPrecomputedPath = '.. /input/precomputedfeaturespetfinder/test_precomputed.csv'
submission = test_id.join(pd.DataFrame(test_lgbm, columns=['AdoptionSpeed']))<save_to_csv>
categoricalFeatures = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2', 'Color3',
'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',
'Health', 'State', 'RescuerID', 'AdoptionSpeed']
numericalFeatures = ['Age', 'Quantity', 'Fee', 'PhotoAmt', 'VideoAmt']
submission.to_csv('submission.csv', index=False )<save_to_csv>
train = pd.read_csv(trainPath)
 test = pd.read_csv(testPath)
 train.info()
submission.to_csv('submission.csv', index=False )<set_options>
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121
warnings.filterwarnings('ignore')
 sns.set(style="darkgrid" )<load_from_csv>
img_size = 256
batch_size = 256
train = pd.read_csv(".. /input/train/train.csv")
 test = pd.read_csv(".. /input/test/test.csv" )<merge>
denseNetPath = '.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5'
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=denseNetPath,
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
def generate_groupBY_data(col_name):
l = train.groupby(['AdoptionSpeed', col_name])[['PetID']].count().reset_index().rename(
 columns={'PetID':'count'})
 count_pets = train.groupby(['AdoptionSpeed', col_name])[['PetID']].count().reset_index().groupby(
 [col_name] ).sum() [['PetID']].reset_index()
new_col_name = 'total_pets_by' + col_name
count_pets.rename(columns={'PetID': new_col_name}, inplace=True)
 temp = l.merge(count_pets, on=[col_name], how='left')
 temp['fraction'] = temp['count'] * 100 / temp[new_col_name]
temp = temp.pivot("AdoptionSpeed", col_name, "fraction")
 return temp<data_type_conversions>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(trainImagePath, pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train['Name'].fillna('None', inplace=True)
 train['Description'].fillna('None', inplace=True)
 test['Name_absent'] = test['Name'].isnull()
test['Name_absent'] = test['Name_absent'].astype(int)
 test['Description_None'] = test['Description'].isnull()
test['Description_None'] = test['Description_None'].astype(int)
 test['Name'].fillna('None', inplace=True)
 test['Description'].fillna('None', inplace=True )<create_dataframe>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
tfidf_vectorizer = TfidfVectorizer(max_features=100, stop_words='english', use_idf=True)
 tfidf_matrix = tfidf_vectorizer.fit_transform(list(train['Description'].values)+
list(test['Description'].values))
cols = tfidf_vectorizer.get_feature_names()
tfidf_train = pd.DataFrame(tfidf_matrix.toarray() [0:train.shape[0],], columns=cols)
 tfidf_test = pd.DataFrame(tfidf_matrix.toarray() [train.shape[0]:,], columns=cols )<concatenate>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(testImagePath, pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train = pd.concat([train, tfidf_train], axis=1)
 test = pd.concat([test, tfidf_test], axis=1 )<data_type_conversions>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
categorical_cols = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2', 'Color3', 'MaturitySize', 'FurLength',
'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'State', 'Name_absent', 'Description_None']
categorical_features_lgbm = []
for col in categorical_cols:
categorical_features_lgbm.append('name:' + col)
 train[col] = train[col].astype('category')
 test[col] = test[col].astype('category' )<prepare_x_and_y>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
X_train = train.drop(['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'], axis=1)
 y_train = train['AdoptionSpeed']
X_test = test.drop(['Name', 'RescuerID', 'Description'], axis=1 )<init_hyperparams>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
param = {'num_leaves': 38,
'min_data_in_leaf': 146,
'objective':'multiclass',
'num_class': 5,
'max_depth': 4,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9980062052116254,
"bagging_freq": 1,
"bagging_fraction": 0.844212672233457,
"bagging_seed": 11,
"metric": 'multi_logloss',
"lambda_l1": 0.12757257166471625,
"random_state": 133,
"verbosity": -1}
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(X_train))
start = time.time()
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train.values, y_train.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(X_train.iloc[trn_idx],
label=y_train.iloc[trn_idx] 
)
 val_data = lgb.Dataset(X_train.iloc[val_idx],
label=y_train.iloc[val_idx] 
)
 num_round = 10000
clf = lgb.train(param,
trn_data,
10000,
valid_sets = [trn_data, val_data],
verbose_eval=500,
early_stopping_rounds = 200)
 oof[val_idx] =(pd.DataFrame(clf.predict(X_train.iloc[val_idx],
num_iteration=clf.best_iteration)).idxmax(axis=1))
print("CV score: {:<8.5f}".format(cohen_kappa_score(oof, y_train, weights="quadratic")) )<save_to_csv>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
p =(pd.DataFrame(clf.predict(X_test.drop(['PetID'], axis=1),
num_iteration=clf.best_iteration)).idxmax(axis=1))
test['AdoptionSpeed'] = p
test[['PetID', 'AdoptionSpeed']].to_csv("submission_lgbm_bayes_optimization.csv", index=False )<import_modules>
labels_breed = pd.read_csv(breedPath)
 labels_state = pd.read_csv(colorPath)
 labels_color = pd.read_csv(statePath )
import os
import numpy as np
import pandas as pd
import json
from pandas.io.json import json_normalize
from pprint import pprint
import matplotlib.pyplot as plt
import seaborn as sns
from PIL import Image
from IPython.display import display, HTML<set_options>
train_image_files = sorted(glob.glob(trainImagePath + '*.jpg'))
train_metadata_files = sorted(glob.glob(trainMetadataPath + '*.json'))
train_sentiment_files = sorted(glob.glob(trainSentimentPath + '*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob(testImagePath + '*.jpg'))
test_metadata_files = sorted(glob.glob(testMetadataPath + '*.json'))
test_sentiment_files = sorted(glob.glob(testSentimentPath + '*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
plt.rcParams['figure.figsize'] =(15, 7)
 pd.set_option("display.max_columns", 400)
 pd.options.display.max_colwidth = 250
pd.set_option("display.max_rows", 100)
 %config InlineBackend.figure_format = 'retina'
sns.set()<load_from_csv>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
df_train = pd.read_csv(".. /input/train/train.csv")
 df_test = pd.read_csv(".. /input/test/test.csv")
 print(f"train.csv shape is {df_train.shape}")
 print(f"test.csv shape is {df_test.shape}" )<load_from_csv>
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
df_breed_labels = pd.read_csv(os.path.join(base_path_data, "breed_labels.csv"))
df_color_labels = pd.read_csv(os.path.join(base_path_data, "color_labels.csv"))
df_state_labels = pd.read_csv(os.path.join(base_path_data, "state_labels.csv"))<load_from_disk>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
with open(os.path.join(base_path_data, "train_sentiment", "048cd8bc0.json")) as f:
data = json.load(f)
 pprint(data )<load_from_disk>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
with open(os.path.join(base_path_data, "train_metadata", "000fb9572-6.json")) as f:
data = json.load(f)
 pprint(data )<count_unique_values>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
common_names = list(set(df_train.Name.unique() ).intersection(set(df_test.Name.unique())))
 len(common_names )<count_missing_values>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
100 * df_train.isnull().sum() / len(df_train )<count_missing_values>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
100 * df_test.isnull().sum() / len(df_test )<count_duplicates>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
df_train[df_train.duplicated(keep=False)].shape<define_variables>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
cols = [col for col in df_train.columns.ravel() if col not in ["PetID", "Name", "Description", "AdoptionSpeed"]]<count_duplicates>
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )
dups = df_train[df_train[cols].duplicated(keep=False)].sort_values(by=cols)
 print(f"Shape of matrix with duplicated rows not considering petid, name nor description {dups.shape}")
 dups.head(2 )<count_duplicates>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
dups = df_test[df_test[cols].duplicated(keep=False)].sort_values(by=cols)
 print(f"Shape of matrix with duplicated rows not considering petid, name nor description {dups.shape}")
 dups.head(2 )<define_variables>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
feats = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1',
'Color2', 'Color3', 'MaturitySize', 'FurLength', 'Vaccinated',
'Dewormed', 'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed']<count_unique_values>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
for cat_c in feats:
if cat_c == "AdoptionSpeed": continue
nunique = df_train[cat_c].nunique()
print(f'{cat_c}:')
 print(f'{nunique} unique values')
 if nunique < 50:
print(f'
Values:
{100 * df_train[cat_c].value_counts(normalize=True, dropna=False)}')
 fig, ax = plt.subplots(figsize=(12,4))
sns.countplot(x=cat_c, hue="AdoptionSpeed", data=df_train, orient="h")
 ax.xaxis.set_label_text(cat_c,fontdict= {'size':14})
 ax.yaxis.set_label_text("Count",fontdict= {'size':14})
 plt.xticks(rotation=90)
 plt.show()
else:
sns.set_style("whitegrid")
 fig, ax = plt.subplots(figsize=(12,4))
for aspeed in range(5):
sns.distplot(df_train[df_train.AdoptionSpeed == aspeed][cat_c].dropna() , hist=False, rug=False, label="AdoptionSpeed = {}".format(aspeed))
ax.xaxis.set_label_text(cat_c,fontdict= {'size':14})
 plt.xticks(rotation=90)
 plt.legend()
plt.show()<import_modules>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
import numpy as np
import pandas as pd
import os
import xgboost as xgb
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
from scipy import sparse
from sklearn.decomposition import TruncatedSVD
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
import scipy as sp
from sklearn import linear_model
from functools import partial
from sklearn.metrics import cohen_kappa_score, mean_squared_error
from sklearn.metrics import confusion_matrix as sk_cmatrix
from collections import Counter
import json
import lightgbm as lgb<compute_test_metric>
X_train.to_csv('train_precomputed.csv', index=False)
 X_test.to_csv('test_precomputed.csv', index=False )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
def rmse(actual, predicted):
return mean_squared_error(actual, predicted)**0.5<feature_engineering>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
train_desc = df_train.Description.fillna("none" ).values
test_desc = df_test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc)+ list(test_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=180)
 svd.fit(X)
 X = svd.transform(X)
 X_test = svd.transform(X_test )<train_on_grid>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
train_desc = df_train.Description.fillna("none" ).values
test_desc = df_test.Description.fillna("none" ).values
svd_n_components = 200
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=svd_n_components)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(svd_n_components)])
 df_train = pd.concat(( df_train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(svd_n_components)])
 df_test = pd.concat(( df_test, X_test), axis=1 )<load_from_disk>
breedDf = pd.read_csv(breedPath)
 breedDf.head()
def readFile(fn):
file = '.. /input/train_sentiment/'+fn['PetID']+'.json'
if os.path.exists(file):
with open(file)as data_file:
data = json.load(data_file)
 df = json_normalize(data)
 mag = df['documentSentiment.magnitude'].values[0]
score = df['documentSentiment.score'].values[0]
return pd.Series([mag,score],index=['mag','score'])
 else:
return pd.Series([0,0],index=['mag','score'])
 def readTestFile(fn):
file = '.. /input/test_sentiment/' + fn['PetID'] + '.json'
if os.path.exists(file):
with open(file)as data_file:
data = json.load(data_file)
 df = json_normalize(data)
 mag = df['documentSentiment.magnitude'].values[0]
score = df['documentSentiment.score'].values[0]
return pd.Series([mag,score],index=['mag','score'])
 else:
print(f'{file} does not exist')
 return pd.Series([0,0],index=['mag','score'])
 df_train[['SentMagnitude', 'SentScore']] = df_train[['PetID']].apply(lambda x: readFile(x), axis=1)
 df_test[['SentMagnitude', 'SentScore']] = df_test[['PetID']].apply(lambda x: readTestFile(x), axis=1 )<categorify>
colorDf = pd.read_csv(colorPath)
 colorDf.head()
<prepare_x_and_y>
stateDf = pd.read_csv(statePath)
 stateDf.head()
y = df_train.AdoptionSpeed
train = np.hstack(( df_train.drop(['Name', 'Description', 'PetID', 'AdoptionSpeed', 'RescuerID'], axis=1 ).values, X))
test = np.hstack(( df_test.drop(['Name', 'Description', 'PetID', 'RescuerID'], axis=1 ).values, X_test))<prepare_x_and_y>
def cleanTransformDataset(dataset, categoricalFeatures):
breedDf2 = breedDf.set_index('BreedID')
 idx = breedDf2.to_dict()
dataset.Breed1 = dataset.Breed1.map(idx['BreedName'])
 dataset.Breed2 = dataset.Breed2.map(idx['BreedName'])
 colorDf2 = colorDf.set_index('ColorID')
 idx = colorDf2.to_dict()
dataset.Color1 = dataset.Color1.map(idx['ColorName'])
 dataset.Color2 = dataset.Color2.map(idx['ColorName'])
 dataset.Color3 = dataset.Color3.map(idx['ColorName'])
 stateDf2 = stateDf.set_index('StateID')
 idx = stateDf2.to_dict()
dataset.State = dataset.State.map(idx['StateName'])
 dataset.Type = dataset.Type.map({1: 'Dog', 2: 'Cat'})
 dataset.Gender = dataset.Gender.map({1: 'Male', 2: 'Female', 3: 'Mixed'})
 dataset.MaturitySize = dataset.MaturitySize.map({1: 'Small', 2: 'Medium', 3: 'Large', 4: 'Extra Large', 0: 'Not Specified'})
 dataset.FurLength = dataset.FurLength.map({1: 'Short', 2: 'Medium', 3: 'Long', 0: 'Not Specified'})
 dataset.Vaccinated = dataset.Vaccinated.map({1: 'Yes', 2: 'No', 3: 'Not Sure'})
 dataset.Dewormed = dataset.Dewormed.map({1: 'Yes', 2: 'No', 3: 'Not Sure'})
 dataset.Sterilized = dataset.Sterilized.map({1: 'Yes', 2: 'No', 3: 'Not Sure'})
 dataset.Health = dataset.Health.map({1: 'Healthy', 2: 'Minor Injury', 3: 'Serious Injury', 0: 'Not Specified'})
 dataset[categoricalFeatures] = dataset[categoricalFeatures].astype('category')
 return dataset
target = df_train['AdoptionSpeed']
train_id = df_train['PetID']
test_id = df_test['PetID']
df_train.drop(['Name', 'Description', 'PetID', 'AdoptionSpeed', 'RescuerID'], axis=1, inplace=True, errors='ignore')
 df_test.drop(['Name', 'Description', 'PetID', 'RescuerID'], axis=1, inplace=True, errors='ignore' )<find_best_model_class>
train = cleanTransformDataset(train, categoricalFeatures)
 test = cleanTransformDataset(test, list(set(categoricalFeatures)- set(['AdoptionSpeed'])) )
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(df_train, df_test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
def extraFeatures(train):
train['L_Color1'] =(pd.isnull(train['Color3'])& pd.isnull(train['Color2'])& pd.notnull(train['Color1'])).astype(int)
 train['L_Color2'] =(pd.isnull(train['Color3'])& pd.notnull(train['Color2'])& pd.notnull(train['Color1'])).astype(int)
 train['L_Color3'] =(pd.notnull(train['Color3'])& pd.notnull(train['Color2'])& pd.notnull(train['Color1'])).astype(int)
 train['L_Breed1'] =(pd.isnull(train['Breed2'])& pd.notnull(train['Breed1'])).astype(int)
 train['L_Breed2'] =(pd.notnull(train['Breed2'])& pd.notnull(train['Breed1'])).astype(int)
 train['Name_Length']= train['Name'].str.len()
train['L_Name_missing'] =(pd.isnull(train['Name'])).astype(int)
 train['L_Breed1_Siamese'] =(train['Breed1']=='Siamese' ).astype(int)
 train['L_Breed1_Persian']=(train['Breed1']=='Persian' ).astype(int)
 train['L_Breed1_Labrador_Retriever']=(train['Breed1']=='Labrador Retriever' ).astype(int)
 train['L_Breed1_Terrier']=(train['Breed1']=='Terrier' ).astype(int)
 train['L_Breed1_Golden_Retriever ']=(train['Breed1']=='Golden Retriever' ).astype(int)
 train['Description_Length']=train['Description'].str.len()
train['L_Fee_Free'] =(train['Fee']==0 ).astype(int)
 pets_total = train.groupby(['RescuerID'] ).size().reset_index(name='N_pets_total')
 train= pd.merge(train, pets_total, left_on='RescuerID', right_on='RescuerID', how='inner')
 train.count()
train['L_NoPhoto'] =(train['PhotoAmt']==0 ).astype(int)
 train['L_NoVideo'] =(train['VideoAmt']==0 ).astype(int)
 train['Log_Age']= np.log(train.Age + 1)
 train.loc[train['Quantity'] > 5, 'Quantity'] = 5
return train
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
train = extraFeatures(train)
 test = extraFeatures(test )
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
train.isna().sum()
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
train['AgeInterval'] = pd.Series(['0-3', '3-6', '6-12', '12-24', '24-48', '48-120', '>120'], dtype='category')
 train.loc[(train['Age'] >= 0)&(train['Age'] <= 3),'AgeInterval'] = '0-3'
train.loc[(train['Age'] > 3)&(train['Age'] <= 6),'AgeInterval'] = '3-6'
train.loc[(train['Age'] > 6)&(train['Age'] <= 12),'AgeInterval'] = '6-12'
train.loc[(train['Age'] > 12)&(train['Age'] <= 24),'AgeInterval'] = '12-24'
train.loc[(train['Age'] > 24)&(train['Age'] <= 48),'AgeInterval'] = '24-48'
train.loc[(train['Age'] > 48)&(train['Age'] <= 120),'AgeInterval'] = '48-120'
train.loc[train['Age'] > 120,'AgeInterval'] = '>120'
quadratic_weighted_kappa(target, train_predictions)
<compute_test_metric>
train = train[train.Fee <= 1500]
rmse(target, [r[0] for r in results['train']])
<prepare_output>
total = train[train.Fee == 0].size
feeFreePercent = [
train[(train.AdoptionSpeed == 0)&(train.Fee == 0)].size,
train[(train.AdoptionSpeed == 1)&(train.Fee == 0)].size,
train[(train.AdoptionSpeed == 2)&(train.Fee == 0)].size,
train[(train.AdoptionSpeed == 3)&(train.Fee == 0)].size,
train[(train.AdoptionSpeed == 4)&(train.Fee == 0)].size
] / total
total = train[train.Fee > 0].size
feePaidPercent = [
train[(train.AdoptionSpeed == 0)&(train.Fee > 0)].size,
train[(train.AdoptionSpeed == 1)&(train.Fee > 0)].size,
train[(train.AdoptionSpeed == 2)&(train.Fee > 0)].size,
train[(train.AdoptionSpeed == 3)&(train.Fee > 0)].size,
train[(train.AdoptionSpeed == 4)&(train.Fee > 0)].size
] / total
percents = np.append(feeFreePercent, feePaidPercent)
 types =(['free'] * 5)+(['paid'] * 5)
 feePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 2, hue='type', style='type', markers=True, data=feePercentDf )
submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
total = train[train.Type == 'Dog'].size
dogPercent = [
train[(train.AdoptionSpeed == 0)&(train.Type == 'Dog')].size,
train[(train.AdoptionSpeed == 1)&(train.Type == 'Dog')].size,
train[(train.AdoptionSpeed == 2)&(train.Type == 'Dog')].size,
train[(train.AdoptionSpeed == 3)&(train.Type == 'Dog')].size,
train[(train.AdoptionSpeed == 4)&(train.Type == 'Dog')].size
] / total
total = train[train.Type == 'Cat'].size
catPercent = [
train[(train.AdoptionSpeed == 0)&(train.Type == 'Cat')].size,
train[(train.AdoptionSpeed == 1)&(train.Type == 'Cat')].size,
train[(train.AdoptionSpeed == 2)&(train.Type == 'Cat')].size,
train[(train.AdoptionSpeed == 3)&(train.Type == 'Cat')].size,
train[(train.AdoptionSpeed == 4)&(train.Type == 'Cat')].size
] / total
percents = np.append(dogPercent, catPercent)
 types =(['dog'] * 5)+(['cat'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 2, hue='type', style='type', markers=True, data=typePercentDf )
submission.to_csv('submission.csv', index=False )<save_to_csv>
strayCat = ['Domestic Short Hair', 'Domestic Medium Hair', 'Domestic Long Hair']
strayDog = ['Mixed Breed']
train.loc[(train.Breed1.isin(strayDog)) &(train.Type == 'Dog'), 'Breed1Type'] = 'Stray'
train.loc[(train.Breed1.isin(strayCat)) &(train.Type == 'Cat'), 'Breed1Type'] = 'Stray'
train.loc[(~train.Breed1.isin(strayDog)) &(train.Type == 'Dog'), 'Breed1Type'] = 'Breed'
train.loc[(~train.Breed1.isin(strayCat)) &(train.Type == 'Cat'), 'Breed1Type'] = 'Breed'
submission.to_csv('submission.csv', index=False )<install_modules>
total = train[(train.Type == 'Dog')&(train.Breed1Type == 'Stray')].size
dogPercentStray = [
train[(train.AdoptionSpeed == 0)&(train.Type == 'Dog')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 1)&(train.Type == 'Dog')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 2)&(train.Type == 'Dog')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 3)&(train.Type == 'Dog')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 4)&(train.Type == 'Dog')&(train.Breed1Type == 'Stray')].size
] / total
total = train[(train.Type == 'Dog')&(train.Breed1Type == 'Breed')].size
dogPercentBreed = [
train[(train.AdoptionSpeed == 0)&(train.Type == 'Dog')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 1)&(train.Type == 'Dog')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 2)&(train.Type == 'Dog')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 3)&(train.Type == 'Dog')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 4)&(train.Type == 'Dog')&(train.Breed1Type == 'Breed')].size
] / total
total = train[(train.Type == 'Cat')&(train.Breed1Type == 'Stray')].size
catPercentStray = [
train[(train.AdoptionSpeed == 0)&(train.Type == 'Cat')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 1)&(train.Type == 'Cat')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 2)&(train.Type == 'Cat')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 3)&(train.Type == 'Cat')&(train.Breed1Type == 'Stray')].size,
train[(train.AdoptionSpeed == 4)&(train.Type == 'Cat')&(train.Breed1Type == 'Stray')].size
] / total
total = train[(train.Type == 'Cat')&(train.Breed1Type == 'Breed')].size
catPercentBreed = [
train[(train.AdoptionSpeed == 0)&(train.Type == 'Cat')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 1)&(train.Type == 'Cat')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 2)&(train.Type == 'Cat')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 3)&(train.Type == 'Cat')&(train.Breed1Type == 'Breed')].size,
train[(train.AdoptionSpeed == 4)&(train.Type == 'Cat')&(train.Breed1Type == 'Breed')].size
] / total
percents = np.append(dogPercentStray, dogPercentBreed)
 percents = np.append(percents, catPercentStray)
 percents = np.append(percents, catPercentBreed)
 types =(['dog-stray'] * 5 + ['dog-breed'] * 5 + ['cat-stray'] * 5 + ['cat-breed'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 4, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
package_path = '.. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2<define_variables>
total = train[(train.MaturitySize == 'Large')].size
largePercent = [
train[(train.AdoptionSpeed == 0)&(train.MaturitySize == 'Large')].size,
train[(train.AdoptionSpeed == 1)&(train.MaturitySize == 'Large')].size,
train[(train.AdoptionSpeed == 2)&(train.MaturitySize == 'Large')].size,
train[(train.AdoptionSpeed == 3)&(train.MaturitySize == 'Large')].size,
train[(train.AdoptionSpeed == 4)&(train.MaturitySize == 'Large')].size
] / total
total = train[(train.MaturitySize == 'Medium')].size
mediumPercent = [
train[(train.AdoptionSpeed == 0)&(train.MaturitySize == 'Medium')].size,
train[(train.AdoptionSpeed == 1)&(train.MaturitySize == 'Medium')].size,
train[(train.AdoptionSpeed == 2)&(train.MaturitySize == 'Medium')].size,
train[(train.AdoptionSpeed == 3)&(train.MaturitySize == 'Medium')].size,
train[(train.AdoptionSpeed == 4)&(train.MaturitySize == 'Medium')].size
] / total
total = train[(train.MaturitySize == 'Small')].size
smallPercent = [
train[(train.AdoptionSpeed == 0)&(train.MaturitySize == 'Small')].size,
train[(train.AdoptionSpeed == 1)&(train.MaturitySize == 'Small')].size,
train[(train.AdoptionSpeed == 2)&(train.MaturitySize == 'Small')].size,
train[(train.AdoptionSpeed == 3)&(train.MaturitySize == 'Small')].size,
train[(train.AdoptionSpeed == 4)&(train.MaturitySize == 'Small')].size
] / total
percents = np.append(largePercent, mediumPercent)
 percents = np.append(percents, smallPercent)
 types =(['Large'] * 5 + ['Medium'] * 5 + ['Small'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 3, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
CFG = {
'train': False,
'train_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/train',
'test_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/test',
'cv_fold_path': '.. /input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',
'train_path': '.. /input/rsna-str-pulmonary-embolism-detection/train.csv',
'test_path': '.. /input/rsna-str-pulmonary-embolism-detection/test.csv',
'image_target_cols': [
'pe_present_on_image',
],
'exam_target_cols': [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
],
'img_num': 200,
'img_size': 256,
'lr': 0.001,
'epochs': 4,
'device': 'cuda',
'train_bs': 2,
'accum_iter': 8,
'verbose_step': 1,
'num_workers': 4,
'efbnet': 'efficientnet-b0',
'train_folds': [np.arange(0,16),
np.concatenate([np.arange(0,12), np.arange(16,20)]),
np.concatenate([np.arange(0,8), np.arange(12,20)]),
np.concatenate([np.arange(0,4), np.arange(8,20)]),
np.arange(4,20),
],
'valid_folds': [np.arange(16,20),
np.arange(12,16),
np.arange(8,12),
np.arange(4,8),
np.arange(0,4)
 ],
'model_path': '.. /input/kh-rsna-model',
'tag': 'efb0_stage2_multilabel'
}
SEED = 42321
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def window(img, WL=50, WW=350):
upper, lower = WL+WW//2, WL-WW//2
X = np.clip(img.copy() , lower, upper)
 X = X - np.min(X)
 X = X / np.max(X)
 return X
def get_img(path):
d = pydicom.read_file(path)


 img =(d.pixel_array * d.RescaleSlope)+ d.RescaleIntercept
r = window(img, -600, 1500)
 g = window(img, 100, 700)
 b = window(img, 40, 400)
 res = np.concatenate([r[:, :, np.newaxis],
g[:, :, np.newaxis],
b[:, :, np.newaxis]], axis=-1)
 res = zoom(res, [CFG['img_size']/res.shape[0], CFG['img_size']/res.shape[1], 1.], prefilter=False, order=1)
 return res
class RSNADatasetStage1(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df[CFG['image_target_cols']].values[index]
path = "{}/{}/{}/{}.dcm".format(self.data_root,
self.df.iloc[index]['StudyInstanceUID'],
self.df.iloc[index]['SeriesInstanceUID'],
self.df.iloc[index]['SOPInstanceUID'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
target = np.clip(target, self.label_smoothing, 1 - self.label_smoothing)
 return img, target
else:
return img
class RSNADataset(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df
self.patients = self.df['StudyInstanceUID'].unique()
self.image_subsampling = image_subsampling
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def get_patients(self):
return self.patients
def __len__(self):
return len(self.patients)
 def __getitem__(self, index: int):
patient = self.patients[index]
df_ = self.df.loc[self.df.StudyInstanceUID == patient]
per_image_feats = get_stage1_columns()
if self.image_subsampling:
img_num = min(CFG['img_num'], df_.shape[0])
 img_ix = np.random.choice(np.arange(df_.shape[0]), replace=False, size=img_num)
 imgs = np.zeros(( CFG['img_num'],), np.float32)
 per_image_preds = np.zeros(( CFG['img_num'], len(per_image_feats)) , np.float32)
 locs = np.zeros(( CFG['img_num'],), np.float32)
 image_masks = np.zeros(( CFG['img_num'],), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( CFG['img_num'], len(CFG['image_target_cols'])) , np.float32)
 else:
img_num = df_.shape[0]
img_ix = np.arange(df_.shape[0])
 imgs = np.zeros(( img_num,), np.float32)
 per_image_preds = np.zeros(( img_num, len(per_image_feats)) , np.float32)
 locs = np.zeros(( img_num,), np.float32)
 image_masks = np.zeros(( img_num,), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( img_num, len(CFG['image_target_cols'])) , np.float32)
 for i, im_ix in enumerate(img_ix):
path = "{}/{}/{}/{}.dcm".format(self.data_root,
df_['StudyInstanceUID'].values[im_ix],
df_['SeriesInstanceUID'].values[im_ix],
df_['SOPInstanceUID'].values[im_ix])
 d = pydicom.read_file(path)
 locs[i] = d.ImagePositionPatient[2]
per_image_preds[i,:] = df_[per_image_feats].values[im_ix,:]
if self.output_label == True:
image_labels[i] = df_[CFG['image_target_cols']].values[im_ix]
seq_ix = np.argsort(locs)

locs = locs[seq_ix]
locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]
locs[0] = 0
per_image_preds = per_image_preds[seq_ix]
if self.output_label == True:
image_labels = image_labels[seq_ix]
image_labels = np.clip(image_labels, self.label_smoothing, 1 - self.label_smoothing)
 exam_label = np.clip(exam_label, self.label_smoothing, 1 - self.label_smoothing)
 return imgs, per_image_preds, locs, image_labels, exam_label, image_masks
else:
return imgs, per_image_preds, locs, img_num, index, seq_ix
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout 
)
 def get_train_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)

 def get_valid_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)

 class RNSAImageFeatureExtractor(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = EfficientNet.from_name(CFG['efbnet'])
 self.pooling = nn.AdaptiveAvgPool2d(1)
 def get_dim(self):
return self.cnn_model._fc.in_features
def forward(self, x):
feats = self.cnn_model.extract_features(x)
 return self.pooling(feats ).view(x.shape[0], -1)
 class RSNAImgClassifierSingle(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 1)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class RSNAImgClassifier(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 9)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class TimeDistributed(nn.Module):
def __init__(self, module, batch_first=True):
super(TimeDistributed, self ).__init__()
self.module = module
self.batch_first = batch_first
def forward(self, x):

 x_size= x.size()
c_in = x.contiguous().view(x_size[0] * x_size[1], *x_size[2:])
 c_out = self.module(c_in)
 r_in = c_out.view(x_size[0], x_size[1], -1)
 if self.batch_first is False:
r_in = r_in.permute(1, 0, 2)
 return r_in
class RSNAClassifier(nn.Module):
def __init__(self, hidden_size=64):
super().__init__()
self.gru = nn.GRU(len(get_stage1_columns())+1, hidden_size, bidirectional=True, batch_first=True, num_layers=2)
 self.image_predictors = TimeDistributed(nn.Linear(hidden_size*2, 1))
self.exam_predictor = nn.Linear(hidden_size*2*2, 9)
 def forward(self, img_preds, locs):
embeds = torch.cat([img_preds, locs.view(locs.shape[0], locs.shape[1], 1)], dim=2)
 embeds, _ = self.gru(embeds)
 image_preds = self.image_predictors(embeds)
 avg_pool = torch.mean(embeds, 1)
 max_pool, _ = torch.max(embeds, 1)
 conc = torch.cat([avg_pool, max_pool], 1)
 exam_pred = self.exam_predictor(conc)
 return image_preds, exam_pred
def rsna_wloss_inference(y_true_img, y_true_exam, y_pred_img, y_pred_exam, chunk_sizes):

 y_true_img, y_true_exam, y_pred_img, y_pred_exam = torch.tensor(y_true_img, dtype=torch.float32), torch.tensor(y_true_exam, dtype=torch.float32), torch.tensor(y_pred_img, dtype=torch.float32), torch.tensor(y_pred_exam, dtype=torch.float32)
 y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks = torch.split(y_true_img, chunk_sizes, dim=0), torch.split(y_true_exam, chunk_sizes, dim=0), torch.split(y_pred_img, chunk_sizes, dim=0), torch.split(y_pred_exam, chunk_sizes, dim=0)
 label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1)
 img_w = 0.07361963
bce_func = torch.nn.BCELoss(reduction='none')
 total_loss = torch.tensor(0, dtype=torch.float32)
 total_weights = torch.tensor(0, dtype=torch.float32)
 for i,(y_true_img_, y_true_exam_, y_pred_img_, y_pred_exam_)in enumerate(zip(y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks)) :
exam_loss = bce_func(y_pred_exam_[0, :], y_true_exam_[0, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
image_loss = bce_func(y_pred_img_, y_true_img_)
 img_num = chunk_sizes[i]
qi = torch.sum(y_true_img_)/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss
def rsna_wloss_train(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):
label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1 ).to(device)
 img_w = 0.07361963
bce_func = torch.nn.BCEWithLogitsLoss(reduction='none' ).to(device)
 total_loss = torch.tensor(0, dtype=torch.float32 ).to(device)
 total_weights = torch.tensor(0, dtype=torch.float32 ).to(device)
 for i in range(y_true_img.shape[0]):
exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
img_mask = image_masks[i]
image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:] ).flatten()
image_loss = image_loss*img_mask
img_num = torch.sum(img_mask)
 qi = torch.sum(y_true_img[i,:])/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss, total_loss, total_weights
def rsna_wloss_valid(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):
label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1 ).to(device)
 img_w = 0.07361963
bce_func = torch.nn.BCEWithLogitsLoss(reduction='none' ).to(device)
 total_loss = torch.tensor(0, dtype=torch.float32 ).to(device)
 total_weights = torch.tensor(0, dtype=torch.float32 ).to(device)
 for i in range(y_true_img.shape[0]):
exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
img_mask = image_masks[i]
image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:] ).flatten()
image_loss = image_loss*img_mask
img_num = torch.sum(img_mask)
 qi = torch.sum(y_true_img[i,:])/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss, total_loss, total_weights
def prepare_train_dataloader(train, cv_df, train_fold, valid_fold):
train_patients = cv_df.loc[cv_df.fold.isin(train_fold), 'StudyInstanceUID'].unique()
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
train_ = train.loc[train.StudyInstanceUID.isin(train_patients),:].reset_index(drop=True)
 valid_ = train.loc[train.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)
 train_ds = RSNADataset(train_, 0.0, CFG['train_img_path'], image_subsampling=True, transforms=get_train_transforms() , output_label=True)
 valid_ds = RSNADataset(valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 train_loader = torch.utils.data.DataLoader(
 train_ds,
batch_size=CFG['train_bs'],
pin_memory=True,
drop_last=False,
shuffle=True,
num_workers=CFG['num_workers'], 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 return train_loader, val_loader
def train_one_epoch(epoch, model, device, scaler, optimizer, train_loader):
model.train()
t = time.time()
loss_sum = 0
loss_w_sum = 0
for step,(imgs, per_image_preds, locs, image_labels, exam_label, image_masks)in enumerate(train_loader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
image_masks = image_masks.to(device ).float()
image_labels = image_labels.to(device ).float()
exam_label = exam_label.to(device ).float()
with autocast() :
image_preds, exam_pred = model(per_image_preds, locs)
 loss, total_loss, total_weights = rsna_wloss_train(image_labels, exam_label, image_preds, exam_pred, image_masks, device)
 scaler.scale(loss ).backward()
loss_sum += total_loss.detach().item()
loss_w_sum += total_weights.detach().item()
if(( step + 1)% CFG['accum_iter'] == 0)or(( step + 1)== len(train_loader)) :
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(train_loader)) :
print(
 f'epoch {epoch} train step {step+1}/{len(train_loader)}, ' + \
f'loss: {loss_sum/loss_w_sum:.4f}, ' + \
f'time: {(time.time() - t):.4f}', end= '\r' if(step + 1)!= len(train_loader)else '
' 
)
 def post_process(exam_pred, image_pred):
rv_lv_ratio_lt_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_lt_1')
 rv_lv_ratio_gte_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_gte_1')
 central_pe_ix = CFG['exam_target_cols'].index('central_pe')
 rightsided_pe_ix = CFG['exam_target_cols'].index('rightsided_pe')
 leftsided_pe_ix = CFG['exam_target_cols'].index('leftsided_pe')
 acute_and_chronic_pe_ix = CFG['exam_target_cols'].index('acute_and_chronic_pe')
 chronic_pe_ix = CFG['exam_target_cols'].index('chronic_pe')
 negative_exam_for_pe_ix = CFG['exam_target_cols'].index('negative_exam_for_pe')
 indeterminate_ix = CFG['exam_target_cols'].index('indeterminate')
 has_pe_image = torch.max(image_pred, 1)[0][0] > 0
rv_lv_ratios = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]]
rv_lv_ratios_1_a = nn.functional.softmax(rv_lv_ratios, dim=1)
 rv_lv_ratios_1_a = torch.log(rv_lv_ratios_1_a/(1-rv_lv_ratios_1_a))
exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]] = torch.where(has_pe_image, rv_lv_ratios_1_a, rv_lv_ratios)
 crl_pe = exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]]
has_no_pe = torch.max(crl_pe ,1)[0] <= 0
max_val = torch.max(crl_pe, 1)[0]
crl_pe_1_b = torch.where(crl_pe==max_val, 0.0001-crl_pe+crl_pe, crl_pe)
 exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]] = torch.where(has_pe_image*has_no_pe, crl_pe_1_b, crl_pe)
 ac_pe = exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]]
both_ac_ch = torch.min(ac_pe ,1)[0] > 0
ac_pe_1_c = nn.functional.softmax(ac_pe, dim=1)
 ac_pe_1_c = torch.log(ac_pe_1_c/(1-ac_pe_1_c))
exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(has_pe_image*both_ac_ch, ac_pe_1_c, ac_pe)
 neg_ind = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
neg_ind_1d = torch.clamp(neg_ind, max=0)
 exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(has_pe_image, neg_ind_1d, neg_ind)
 ne_inde = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
ne_inde_2_a = nn.functional.softmax(ne_inde, dim=1)
 ne_inde_2_a = torch.log(ne_inde_2_a/(1-ne_inde_2_a))
exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(~has_pe_image, ne_inde_2_a, ne_inde)
 all_other_exam_labels = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]]
all_other_exam_labels_2_b = torch.clamp(all_other_exam_labels, max=0)
 exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(~has_pe_image, all_other_exam_labels_2_b, all_other_exam_labels)
 return exam_pred, image_pred
def valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=False):
model.eval()
t = time.time()
loss_sum = 0
loss_w_sum = 0
for step,(imgs, per_image_preds, locs, image_labels, exam_label, image_masks)in enumerate(val_loader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
image_masks = image_masks.to(device ).float()
image_labels = image_labels.to(device ).float()
exam_label = exam_label.to(device ).float()
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred, image_preds= post_process(exam_pred, image_preds)
 loss, total_loss, total_weights = rsna_wloss_valid(image_labels, exam_label, image_preds, exam_pred, image_masks, device)
 loss_sum += total_loss.detach().item()
loss_w_sum += total_weights.detach().item()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(val_loader)) :
print(
 f'epoch {epoch} valid Step {step+1}/{len(val_loader)}, ' + \
f'loss: {loss_sum/loss_w_sum:.4f}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(val_loader)else '
' 
)
 if schd_loss_update:
scheduler.step(loss_sum/loss_w_sum)
 else:
scheduler.step()
def check_label_consistency(checking_df):
df = checking_df.copy()
print(df.shape)
 df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('label in-consistency counts:', errors.shape)
 if errors.shape[0] > 0:
print(errors.broken_rule.value_counts())
 print(errors)
 assert False
def inference(model, device, df, root_path):
model.eval()
t = time.time()
ds = RSNADataset(df, 0.0, root_path, image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 dataloader = torch.utils.data.DataLoader(
 ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 patients = ds.get_patients()
res_dfs = []
for step,(imgs, per_image_preds, locs, img_num, index, seq_ix)in enumerate(dataloader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
index = index.detach().numpy() [0]
seq_ix = seq_ix.detach().numpy() [0,:]
patient_filt =(df.StudyInstanceUID == patients[index])
 patient_df = pd.DataFrame()
patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]
patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values
patient_df['StudyInstanceUID'] = patients[index]
for c in CFG['image_target_cols']+CFG['exam_target_cols']:
patient_df[c] = 0.0
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred, image_preds = post_process(exam_pred, image_preds)
 exam_pred = torch.sigmoid(exam_pred ).cpu().detach().numpy()
image_preds = torch.sigmoid(image_preds ).cpu().detach().numpy()
patient_df[CFG['exam_target_cols']] = exam_pred[0]
patient_df[CFG['image_target_cols']] = image_preds[0,:]
res_dfs += [patient_df]


if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(dataloader)) :
print(
 f'Inference Step {step+1}/{len(dataloader)}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(dataloader)else '
' 
)
 res_dfs = pd.concat(res_dfs, axis=0 ).reset_index(drop=True)
 res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')
 print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].head(5))
print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].tail(5))
assert res_dfs.shape[0] == df.shape[0]
check_label_consistency(res_dfs)
 return res_dfs
STAGE1_CFGS = [
{
'tag': 'efb0_stage1',
'model_constructor': RSNAImgClassifierSingle,
'dataset_constructor': RSNADatasetStage1,
'output_len': 1
},
{
'tag': 'efb0_stage1_multilabel',
'model_constructor': RSNAImgClassifier,
'dataset_constructor': RSNADatasetStage1,
'output_len': 9
},
]
STAGE1_CFGS_TAG = 'efb0-stage1-single-multi-label'
def get_stage1_columns() :
new_feats = []
for cfg in STAGE1_CFGS:
for i in range(cfg['output_len']):
f = cfg['tag']+'_'+str(i)
 new_feats += [f]
return new_feats
def update_stage1_oof_preds(df, cv_df):
res_file_name = STAGE1_CFGS_TAG+"-train.csv"
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
if os.path.isfile(res_file_name):
df = pd.read_csv(res_file_name)
 print('img acc:',(( df[new_feats[0]]>0)==df[CFG['image_target_cols'][0]] ).mean())
 return df
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
if fold < 0:
continue
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
filt = df.StudyInstanceUID.isin(valid_patients)
 valid_ = df.loc[filt,:].reset_index(drop=True)
 image_preds_all_list = []
for cfg in STAGE1_CFGS:
valid_ds = cfg['dataset_constructor'](valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(valid_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, cfg['tag'])))
 model.eval()
image_preds_all = []
correct_count = 0
count = 0
for step,(imgs, target)in enumerate(val_loader):
imgs = imgs.to(device ).float()
target = target.to(device ).float()
image_preds = model(imgs)
 if len(image_preds.shape)== 1:
image_preds = image_preds.view(-1, 1)
 correct_count +=(( image_preds[:,0]>0)== target[:,0] ).sum().detach().item()
count += imgs.shape[0]
image_preds_all += [image_preds.cpu().detach().numpy() ]
print('acc: {:.4f}, {}, {}, {}/{}'.format(correct_count/count, correct_count, count, step+1, len(val_loader)) , end='\r')
 print()
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model, val_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[filt, new_feats] = image_preds_all_list
df.to_csv(res_file_name, index=False)
 return df
def update_stage1_test_preds(df):
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
image_preds_all_list = []
for cfg in STAGE1_CFGS:
test_ds = cfg['dataset_constructor'](df, 0.0, CFG['test_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 test_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(test_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], cfg['tag'])))
 model.eval()
image_preds_all = []
for step, imgs in enumerate(tqdm(test_loader)) :
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [image_preds.cpu().detach().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model, test_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[:,new_feats] = image_preds_all_list
return df
if __name__ == '__main__':
if CFG['train']:
seed_everything(SEED)
 if CFG['train']:
train_df = pd.read_csv(CFG['train_path'])
 cv_df = pd.read_csv(CFG['cv_fold_path'])
 with torch.no_grad() :
train_df = update_stage1_oof_preds(train_df,cv_df)
 else:
test_df = pd.read_csv(CFG['test_path'])
 with torch.no_grad() :
test_df = update_stage1_test_preds(test_df)
 if CFG['train']:
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, train_fold, valid_fold)
 device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'])
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False
for epoch in range(CFG['epochs']):
train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)
 with torch.no_grad() :
valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)
 torch.save(model.state_dict() ,'{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))
model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag'])))
 valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
valid_ = train_df.loc[train_df.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)
 with torch.no_grad() :
val_pred_df = inference(model, device, valid_, CFG['train_img_path'])
 target = valid_[CFG['image_target_cols']].values
pred =(val_pred_df[CFG['image_target_cols']].values > 0.5 ).astype(int)
 print('Image PE Accuracy: {:.3f}'.format(( target==pred ).mean() *100))
loss = rsna_wloss_inference(valid_[CFG['image_target_cols']].values, valid_[CFG['exam_target_cols']].values,
val_pred_df[CFG['image_target_cols']].values, val_pred_df[CFG['exam_target_cols']].values,
list(valid_.groupby('StudyInstanceUID', sort=False)['SOPInstanceUID'].count()))
print('Validation loss = {:.4f}'.format(loss.detach().item()))
del model, optimizer, train_loader, val_loader, scaler, scheduler
torch.cuda.empty_cache()
train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, np.arange(0, 20), np.array([]))
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'])
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False
for epoch in range(CFG['epochs']):
train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)
 torch.save(model.state_dict() ,'{}/model_{}'.format(CFG['model_path'], CFG['tag']))
else:
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], CFG['tag'])))
 test_pred_df = inference(model, device, test_df, CFG['test_img_path'])
 test_pred_df.to_csv('kh_submission_raw.csv')
 ids = []
labels = []
gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False ).mean()
for col in CFG['exam_target_cols']:
ids += [[patient+'_'+col for patient in gp_mean.index]]
labels += [gp_mean[col].values]
ids += [test_pred_df.SOPInstanceUID.values]
labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]
ids = np.concatenate(ids)
 labels = np.concatenate(labels)
 assert len(ids)== len(labels)
 submission = pd.DataFrame()
submission['id'] = ids
submission['label'] = labels
print(submission.head(3))
print(submission.tail(3))
print(submission.shape)
 submission.to_csv('submission.csv', index=False )<set_options>
total = train[(train.FurLength == 'Long')].size
longPercent = [
train[(train.AdoptionSpeed == 0)&(train.FurLength == 'Long')].size,
train[(train.AdoptionSpeed == 1)&(train.FurLength == 'Long')].size,
train[(train.AdoptionSpeed == 2)&(train.FurLength == 'Long')].size,
train[(train.AdoptionSpeed == 3)&(train.FurLength == 'Long')].size,
train[(train.AdoptionSpeed == 4)&(train.FurLength == 'Long')].size
] / total
total = train[(train.FurLength == 'Medium')].size
mediumPercent = [
train[(train.AdoptionSpeed == 0)&(train.FurLength == 'Medium')].size,
train[(train.AdoptionSpeed == 1)&(train.FurLength == 'Medium')].size,
train[(train.AdoptionSpeed == 2)&(train.FurLength == 'Medium')].size,
train[(train.AdoptionSpeed == 3)&(train.FurLength == 'Medium')].size,
train[(train.AdoptionSpeed == 4)&(train.FurLength == 'Medium')].size
] / total
total = train[(train.FurLength == 'Short')].size
shortPercent = [
train[(train.AdoptionSpeed == 0)&(train.FurLength == 'Short')].size,
train[(train.AdoptionSpeed == 1)&(train.FurLength == 'Short')].size,
train[(train.AdoptionSpeed == 2)&(train.FurLength == 'Short')].size,
train[(train.AdoptionSpeed == 3)&(train.FurLength == 'Short')].size,
train[(train.AdoptionSpeed == 4)&(train.FurLength == 'Short')].size
] / total
percents = np.append(longPercent, mediumPercent)
 percents = np.append(percents, shortPercent)
 types =(['Long'] * 5 + ['Medium'] * 5 + ['Short'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 3, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
warnings.filterwarnings('ignore' )<define_variables>
total = train[(train.Vaccinated == 'Yes')].size
yesPercent = [
train[(train.AdoptionSpeed == 0)&(train.Vaccinated == 'Yes')].size,
train[(train.AdoptionSpeed == 1)&(train.Vaccinated == 'Yes')].size,
train[(train.AdoptionSpeed == 2)&(train.Vaccinated == 'Yes')].size,
train[(train.AdoptionSpeed == 3)&(train.Vaccinated == 'Yes')].size,
train[(train.AdoptionSpeed == 4)&(train.Vaccinated == 'Yes')].size
] / total
total = train[(train.Vaccinated == 'No')].size
noPercent = [
train[(train.AdoptionSpeed == 0)&(train.Vaccinated == 'No')].size,
train[(train.AdoptionSpeed == 1)&(train.Vaccinated == 'No')].size,
train[(train.AdoptionSpeed == 2)&(train.Vaccinated == 'No')].size,
train[(train.AdoptionSpeed == 3)&(train.Vaccinated == 'No')].size,
train[(train.AdoptionSpeed == 4)&(train.Vaccinated == 'No')].size
] / total
total = train[(train.Vaccinated == 'Not Sure')].size
notSurePercent = [
train[(train.AdoptionSpeed == 0)&(train.Vaccinated == 'Not Sure')].size,
train[(train.AdoptionSpeed == 1)&(train.Vaccinated == 'Not Sure')].size,
train[(train.AdoptionSpeed == 2)&(train.Vaccinated == 'Not Sure')].size,
train[(train.AdoptionSpeed == 3)&(train.Vaccinated == 'Not Sure')].size,
train[(train.AdoptionSpeed == 4)&(train.Vaccinated == 'Not Sure')].size
] / total
percents = np.append(yesPercent, noPercent)
 percents = np.append(percents, notSurePercent)
 types =(['Yes'] * 5 + ['No'] * 5 + ['Not Sure'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 3, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )<predict_on_test>
total = train[(train.Dewormed == 'Yes')].size
yesPercent = [
train[(train.AdoptionSpeed == 0)&(train.Dewormed == 'Yes')].size,
train[(train.AdoptionSpeed == 1)&(train.Dewormed == 'Yes')].size,
train[(train.AdoptionSpeed == 2)&(train.Dewormed == 'Yes')].size,
train[(train.AdoptionSpeed == 3)&(train.Dewormed == 'Yes')].size,
train[(train.AdoptionSpeed == 4)&(train.Dewormed == 'Yes')].size
] / total
total = train[(train.Dewormed == 'No')].size
noPercent = [
train[(train.AdoptionSpeed == 0)&(train.Dewormed == 'No')].size,
train[(train.AdoptionSpeed == 1)&(train.Dewormed == 'No')].size,
train[(train.AdoptionSpeed == 2)&(train.Dewormed == 'No')].size,
train[(train.AdoptionSpeed == 3)&(train.Dewormed == 'No')].size,
train[(train.AdoptionSpeed == 4)&(train.Dewormed == 'No')].size
] / total
total = train[(train.Dewormed == 'Not Sure')].size
notSurePercent = [
train[(train.AdoptionSpeed == 0)&(train.Dewormed == 'Not Sure')].size,
train[(train.AdoptionSpeed == 1)&(train.Dewormed == 'Not Sure')].size,
train[(train.AdoptionSpeed == 2)&(train.Dewormed == 'Not Sure')].size,
train[(train.AdoptionSpeed == 3)&(train.Dewormed == 'Not Sure')].size,
train[(train.AdoptionSpeed == 4)&(train.Dewormed == 'Not Sure')].size
] / total
percents = np.append(yesPercent, noPercent)
 percents = np.append(percents, notSurePercent)
 types =(['Yes'] * 5 + ['No'] * 5 + ['Not Sure'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 3, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
def make_predictions(tr_df, tt_df, features_columns, target, lgb_params, NFOLDS=2):
folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 X,y = tr_df[features_columns], tr_df[target]
P,P_y = tt_df[features_columns], tt_df[target]
tt_df = tt_df[['TransactionID',target]]
predictions = np.zeros(len(tt_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]
print(len(tr_x),len(vl_x))
tr_data = lgb.Dataset(tr_x, label=tr_y)
 if LOCAL_TEST:
vl_data = lgb.Dataset(P, label=P_y)
 else:
vl_data = lgb.Dataset(vl_x, label=vl_y)
 estimator = lgb.train(
 lgb_params,
tr_data,
valid_sets = [tr_data, vl_data],
verbose_eval = 200, 
)
 pp_p = estimator.predict(P)
 predictions += pp_p/NFOLDS
if LOCAL_TEST:
feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance() ,X.columns)) , columns=['Value','Feature'])
 print(feature_imp)
 del tr_x, tr_y, vl_x, vl_y, tr_data, vl_data
gc.collect()
tt_df['prediction'] = predictions
return tt_df 
<define_variables>
total = train[(train.Sterilized == 'Yes')].size
yesPercent = [
train[(train.AdoptionSpeed == 0)&(train.Sterilized == 'Yes')].size,
train[(train.AdoptionSpeed == 1)&(train.Sterilized == 'Yes')].size,
train[(train.AdoptionSpeed == 2)&(train.Sterilized == 'Yes')].size,
train[(train.AdoptionSpeed == 3)&(train.Sterilized == 'Yes')].size,
train[(train.AdoptionSpeed == 4)&(train.Sterilized == 'Yes')].size
] / total
total = train[(train.Sterilized == 'No')].size
noPercent = [
train[(train.AdoptionSpeed == 0)&(train.Sterilized == 'No')].size,
train[(train.AdoptionSpeed == 1)&(train.Sterilized == 'No')].size,
train[(train.AdoptionSpeed == 2)&(train.Sterilized == 'No')].size,
train[(train.AdoptionSpeed == 3)&(train.Sterilized == 'No')].size,
train[(train.AdoptionSpeed == 4)&(train.Sterilized == 'No')].size
] / total
total = train[(train.Sterilized == 'Not Sure')].size
notSurePercent = [
train[(train.AdoptionSpeed == 0)&(train.Sterilized == 'Not Sure')].size,
train[(train.AdoptionSpeed == 1)&(train.Sterilized == 'Not Sure')].size,
train[(train.AdoptionSpeed == 2)&(train.Sterilized == 'Not Sure')].size,
train[(train.AdoptionSpeed == 3)&(train.Sterilized == 'Not Sure')].size,
train[(train.AdoptionSpeed == 4)&(train.Sterilized == 'Not Sure')].size
] / total
percents = np.append(yesPercent, noPercent)
 percents = np.append(percents, notSurePercent)
 types =(['Yes'] * 5 + ['No'] * 5 + ['Not Sure'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 3, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )<feature_engineering>
total = train[(train.Health == 'Healthy')].size
healthyPercent = [
train[(train.AdoptionSpeed == 0)&(train.Health == 'Healthy')].size,
train[(train.AdoptionSpeed == 1)&(train.Health == 'Healthy')].size,
train[(train.AdoptionSpeed == 2)&(train.Health == 'Healthy')].size,
train[(train.AdoptionSpeed == 3)&(train.Health == 'Healthy')].size,
train[(train.AdoptionSpeed == 4)&(train.Health == 'Healthy')].size
] / total
total = train[(train.Health == 'Minor Injury')].size
minorPercent = [
train[(train.AdoptionSpeed == 0)&(train.Health == 'Minor Injury')].size,
train[(train.AdoptionSpeed == 1)&(train.Health == 'Minor Injury')].size,
train[(train.AdoptionSpeed == 2)&(train.Health == 'Minor Injury')].size,
train[(train.AdoptionSpeed == 3)&(train.Health == 'Minor Injury')].size,
train[(train.AdoptionSpeed == 4)&(train.Health == 'Minor Injury')].size
] / total
total = train[(train.Health == 'Serious Injury')].size
seriousPercent = [
train[(train.AdoptionSpeed == 0)&(train.Health == 'Serious Injury')].size,
train[(train.AdoptionSpeed == 1)&(train.Health == 'Serious Injury')].size,
train[(train.AdoptionSpeed == 2)&(train.Health == 'Serious Injury')].size,
train[(train.AdoptionSpeed == 3)&(train.Health == 'Serious Injury')].size,
train[(train.AdoptionSpeed == 4)&(train.Health == 'Serious Injury')].size
] / total
percents = np.append(healthyPercent, minorPercent)
 percents = np.append(percents, seriousPercent)
 types =(['Healthy'] * 5 + ['Minor Injury'] * 5 + ['Serious Injury'] * 5)
 typePercentDf = pd.DataFrame({ 'percent': percents, 'type': types })
 fig, ax = plt.subplots(figsize=(8,8))
sns.lineplot(y='percent', x=[0, 1, 2, 3, 4] * 3, hue='type', style='type', markers=True, data=typePercentDf, palette='coolwarm' )
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 if LOCAL_TEST:
train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max() -1)].reset_index(drop=True)
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = train_identity[train_identity['TransactionID'].isin(
 test_df['TransactionID'])].reset_index(drop=True)
 train_identity = train_identity[train_identity['TransactionID'].isin(
 train_df['TransactionID'])].reset_index(drop=True)
 del train_df['DT_M'], test_df['DT_M']
else:
test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl')
 base_columns = list(train_df)+ list(train_identity)
 print('Shape control:', train_df.shape, test_df.shape )<feature_engineering>
target = 'AdoptionSpeed'
categorical = ['Type', 'Breed1', 'Gender', 'Color1', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'State']
features = categorical + numericalFeatures
data = train[features + [target]].dropna()
i_cols = ['card1']
for col in i_cols:
valid_card = pd.concat([train_df[[col]], test_df[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card = valid_card[valid_card>2]
valid_card = list(valid_card.index)
 train_df[col] = np.where(train_df[col].isin(valid_card), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(valid_card), test_df[col], np.nan )<categorify>
def generateConfusionMatrix(y_real, y_predicted):
cm = pd.DataFrame()
cm['Satisfaction'] = y_real
cm['Predict'] = y_predicted
mappingSatisfaction = {0:'Same Day', 1: 'First Week', 2: 'First Month', 3: '2-3 Month', 4: 'Non-Adopted >100'}
mappingPredict = {0.0:'Same Day', 1.0: 'First Week', 2.0: 'First Month', 3.0: '2-3 Month', 4.0: 'Non-Adopted >100'}
cm = cm.replace({'Satisfaction': mappingSatisfaction, 'Predict': mappingPredict})
 return pd.crosstab(cm['Satisfaction'], cm['Predict'], margins=True)
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8','D9',
'addr1','addr2',
'dist1','dist2',
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 valid_card = temp_df[col].value_counts().to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(valid_card)
 test_df[col+'_fq_enc'] = test_df[col].map(valid_card )<categorify>
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col+'_target_mean'] = train_df[col].map(temp_dict)
 test_df[col+'_target_mean'] = test_df[col].map(temp_dict )<data_type_conversions>
def calculateClassificationScores(y_true, y_predicted, model, X_test, average='macro'):
accuracy = accuracy_score(y_true, y_predicted)
 f1 = f1_score(y_true, y_predicted, average=average)
 precision = precision_score(y_true, y_predicted, average=average)
 recall = recall_score(y_true, y_predicted, average=average)
 if model and isinstance(model, CatBoostClassifier):
score = model.get_best_score()
if score.get('validation_0'):
multiclass = score['validation_0']['MultiClass']
else:
multiclass = model.score(X_test, y_true)
 return(accuracy, f1, precision, recall, multiclass)
 return(accuracy, f1, precision, recall)
 def crossValidation(params, X, y):
f1_scores = []
accuracy_scores = []
precision_scores = []
recall_scores = []
multiclass_scores = []
for train_index, val_index in skf.split(X.values, y.values):
X_train = X[X.index.isin(train_index)]
X_train = FeaturesData(
 num_feature_data=X_train[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_train[categoricalFeatures].__array__(dtype=object)
)
 y_train = y[y.index.isin(train_index)].astype('int' ).values
X_valid = X[X.index.isin(val_index)]
X_valid = FeaturesData(
 num_feature_data=X_valid[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_valid[categoricalFeatures].__array__(dtype=object)
)
 y_valid = y[y.index.isin(val_index)].astype('int' ).values
pool_test = Pool(X_valid, label=y_valid)
 clf = CatBoostClassifier(
 loss_function='MultiClass',
verbose=False,
depth=params['depth'],
iterations=params['iterations'],
l2_leaf_reg=params['l2_leaf_reg'],
learning_rate=params['learning_rate'],
task_type='CPU' 
)
 clf.fit(X_train, y_train, eval_set=pool_test, use_best_model=True)
 y_pred = clf.predict(X_valid)
(accuracy, f1, precision, recall, multiclass)= calculateClassificationScores(y_valid, y_pred, clf, X_valid)
 multiclass_scores.append(multiclass)
 accuracy_scores.append(accuracy)
 f1_scores.append(f1)
 precision_scores.append(precision)
 recall_scores.append(recall)
 return(multiclass_scores, accuracy_scores, f1_scores, precision_scores, recall_scores)
 def searchBestParams(grid, X, y):
catboostDf = pd.DataFrame({
'model':[],
'multiclass_score_mean':[],
'multiclass_score_std':[],
'f1_score_mean':[],
'f1_score_std':[],
'accuracy_score_mean':[],
'accuracy_score_std':[],
'precision_score_mean':[],
'precision_score_std':[],
'recall_score_mean':[],
'recall_score_std':[],
'params': []} 
)
 for params in grid:
print(params)
(multiclass_scores, accuracy_scores, f1_scores, precision_scores, recall_scores)= crossValidation(params, X, y)
 catboostDf = catboostDf.append({
'multiclass_score_mean': np.mean(multiclass_scores),
'multiclass_score_std': np.std(multiclass_scores),
'f1_score_mean': np.mean(f1_scores),
'f1_score_std': np.std(f1_scores),
'accuracy_score_mean': np.mean(accuracy_scores),
'accuracy_score_std': np.std(accuracy_scores),
'precision_score_mean': np.mean(precision_scores),
'precision_score_std': np.std(precision_scores),
'recall_score_mean': np.mean(recall_scores),
'recall_score_std': np.std(recall_scores),
'params': params
}, ignore_index=True)
 return catboostDf
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
for df in [train_df, test_df]:
df['M_sum'] = df[i_cols].sum(axis=1 ).astype(np.int8)
 df['M_na'] = df[i_cols].isna().sum(axis=1 ).astype(np.int8 )<data_type_conversions>
train_df['uid'] = train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)
 test_df['uid'] = test_df['card1'].astype(str)+'_'+test_df['card2'].astype(str)
 train_df['uid2'] = train_df['uid'].astype(str)+'_'+train_df['card3'].astype(str)+'_'+train_df['card5'].astype(str)
 test_df['uid2'] = test_df['uid'].astype(str)+'_'+test_df['card3'].astype(str)+'_'+test_df['card5'].astype(str)
 train_df['uid3'] = train_df['uid2'].astype(str)+'_'+train_df['addr1'].astype(str)+'_'+train_df['addr2'].astype(str)
 test_df['uid3'] = test_df['uid2'].astype(str)+'_'+test_df['addr1'].astype(str)+'_'+test_df['addr2'].astype(str)
 train_df['TransactionAmt_check'] = np.where(train_df['TransactionAmt'].isin(test_df['TransactionAmt']), 1, 0)
 test_df['TransactionAmt_check'] = np.where(test_df['TransactionAmt'].isin(train_df['TransactionAmt']), 1, 0)
 i_cols = ['card1','card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train_df[[col, 'TransactionAmt']], test_df[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df)
 test_df[new_col_name] = test_df[col].map(temp_df )<categorify>
def trainCatboost(numericalFeatures, categoricalFeatures, target, data, params=None):
print(numericalFeatures)
 print(categoricalFeatures)
 print(target)
 print(data.size)
 X_train, X_test, y_train, y_test = train_test_split(
 data[numericalFeatures + categoricalFeatures],
data[target],
test_size=0.25,
random_state=42 
)
 X_train_pool = FeaturesData(
 num_feature_data=X_train[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_train[categoricalFeatures].__array__(dtype=object)
)
 y_train_pool = y_train.astype('int' ).values
X_test_pool = FeaturesData(
 num_feature_data=X_test[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_test[categoricalFeatures].__array__(dtype=object)
)
 y_test_pool = y_test.astype('int' ).values
model = None
if params:
model = CatBoostClassifier(
 loss_function='MultiClass',
verbose=False,
depth=params['depth'],
iterations=params['iterations'],
l2_leaf_reg=params['l2_leaf_reg'],
learning_rate=params['learning_rate'],
task_type='GPU',
class_weights=[4, 1, 1, 1, 1] 
)
 else:
model = CatBoostClassifier(
 loss_function='MultiClass',
verbose=False,
depth=8,
iterations=140,
l2_leaf_reg=12,
learning_rate=0.15,
task_type='GPU',
class_weights=[4, 1, 1, 1, 1] 
)
 model.fit(X_train_pool, y_train_pool,logging_level='Silent')
 y_predicted = model.predict(X_test_pool)
 return(model, y_predicted, X_train, y_train, X_train_pool, X_test_pool, y_test )
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )<define_variables>
rm_cols = [
'TransactionID','TransactionDT',
TARGET,
'uid','uid2','uid3',
]
features_columns = [col for col in list(train_df)if col not in rm_cols]<init_hyperparams>
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':800,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}<predict_on_test>
if LOCAL_TEST:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 20000
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.005
lgb_params['n_estimators'] = 1800
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=10 )<save_to_csv>
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )<load_from_csv>
%%time
train_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
<merge>
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True )<prepare_x_and_y>
def generateSubmissionCatboost(model, numericalFeatures, categoricalFeatures, test, fileName):
X = test[numericalFeatures + categoricalFeatures]
X = FeaturesData(
 num_feature_data=X[numericalFeatures].astype('float32' ).values,
cat_feature_data=X[categoricalFeatures].__array__(dtype=object)
)
 predictions = model.predict(X)
 test['AdoptionSpeed'] = predictions
test.AdoptionSpeed = test['AdoptionSpeed'].map({0.0: '0', 1.0: '1', 2.0: '2', 3.0: '3', 4.0: '4'})
 test[['PetID', 'AdoptionSpeed']].to_csv(fileName + '.csv', index=False )
y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity<prepare_x_and_y>
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy() 
<drop_column>
del train, test<data_type_conversions>
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999 )<categorify>
to_drop_columns = ['PetID', 'Name', 'RescuerID']
X_train_non_null = X_train_non_null.drop(to_drop_columns, axis=1 )
for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<choose_model_class>
testIds = X_test_non_null['PetID']
X_test_non_null = X_test_non_null.drop(to_drop_columns, axis=1 )
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
missing=-999,
random_state=2019,
tree_method='gpu_hist'
)<train_model>
X_test_non_null = X_test_non_null.drop(['AdoptionSpeed'], axis=1 )
%time clf.fit(X_train, y_train )<load_from_csv>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
sample_submission = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID' )<save_to_csv>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
sample_submission['isFraud'] = clf.predict_proba(X_test)[:,1]
sample_submission.to_csv('simple_xgboost.csv' )<define_variables>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
warnings.filterwarnings("ignore")
 SEED = 42
float_cols = [
'TransactionAmt',
'card2',
'card3',
'card5',
'addr1',
'addr2',
'dist1',
'dist2',
'C1',
'C2',
'C3',
'C4',
'C5',
'C6',
'C7',
'C8',
'C9',
'C10',
'C11',
'C12',
'C13',
'C14',
'D1',
'D2',
'D3',
'D4',
'D5',
'D6',
'D7',
'D8',
'D9',
'D10',
'D11',
'D12',
'D13',
'D14',
'D15',
'V1',
'V2',
'V3',
'V4',
'V5',
'V6',
'V7',
'V8',
'V9',
'V10',
'V11',
'V12',
'V13',
'V14',
'V15',
'V16',
'V17',
'V18',
'V19',
'V20',
'V21',
'V22',
'V23',
'V24',
'V25',
'V26',
'V27',
'V28',
'V29',
'V30',
'V31',
'V32',
'V33',
'V34',
'V35',
'V36',
'V37',
'V38',
'V39',
'V40',
'V41',
'V42',
'V43',
'V44',
'V45',
'V46',
'V47',
'V48',
'V49',
'V50',
'V51',
'V52',
'V53',
'V54',
'V55',
'V56',
'V57',
'V58',
'V59',
'V60',
'V61',
'V62',
'V63',
'V64',
'V65',
'V66',
'V67',
'V68',
'V69',
'V70',
'V71',
'V72',
'V73',
'V74',
'V75',
'V76',
'V77',
'V78',
'V79',
'V80',
'V81',
'V82',
'V83',
'V84',
'V85',
'V86',
'V87',
'V88',
'V89',
'V90',
'V91',
'V92',
'V93',
'V94',
'V95',
'V96',
'V97',
'V98',
'V99',
'V100',
'V101',
'V102',
'V103',
'V104',
'V105',
'V106',
'V107',
'V108',
'V109',
'V110',
'V111',
'V112',
'V113',
'V114',
'V115',
'V116',
'V117',
'V118',
'V119',
'V120',
'V121',
'V122',
'V123',
'V124',
'V125',
'V126',
'V127',
'V128',
'V129',
'V130',
'V131',
'V132',
'V133',
'V134',
'V135',
'V136',
'V137',
'V138',
'V139',
'V140',
'V141',
'V142',
'V143',
'V144',
'V145',
'V146',
'V147',
'V148',
'V149',
'V150',
'V151',
'V152',
'V153',
'V154',
'V155',
'V156',
'V157',
'V158',
'V159',
'V160',
'V161',
'V162',
'V163',
'V164',
'V165',
'V166',
'V167',
'V168',
'V169',
'V170',
'V171',
'V172',
'V173',
'V174',
'V175',
'V176',
'V177',
'V178',
'V179',
'V180',
'V181',
'V182',
'V183',
'V184',
'V185',
'V186',
'V187',
'V188',
'V189',
'V190',
'V191',
'V192',
'V193',
'V194',
'V195',
'V196',
'V197',
'V198',
'V199',
'V200',
'V201',
'V202',
'V203',
'V204',
'V205',
'V206',
'V207',
'V208',
'V209',
'V210',
'V211',
'V212',
'V213',
'V214',
'V215',
'V216',
'V217',
'V218',
'V219',
'V220',
'V221',
'V222',
'V223',
'V224',
'V225',
'V226',
'V227',
'V228',
'V229',
'V230',
'V231',
'V232',
'V233',
'V234',
'V235',
'V236',
'V237',
'V238',
'V239',
'V240',
'V241',
'V242',
'V243',
'V244',
'V245',
'V246',
'V247',
'V248',
'V249',
'V250',
'V251',
'V252',
'V253',
'V254',
'V255',
'V256',
'V257',
'V258',
'V259',
'V260',
'V261',
'V262',
'V263',
'V264',
'V265',
'V266',
'V267',
'V268',
'V269',
'V270',
'V271',
'V272',
'V273',
'V274',
'V275',
'V276',
'V277',
'V278',
'V279',
'V280',
'V281',
'V282',
'V283',
'V284',
'V285',
'V286',
'V287',
'V288',
'V289',
'V290',
'V291',
'V292',
'V293',
'V294',
'V295',
'V296',
'V297',
'V298',
'V299',
'V300',
'V301',
'V302',
'V303',
'V304',
'V305',
'V306',
'V307',
'V308',
'V309',
'V310',
'V311',
'V312',
'V313',
'V314',
'V315',
'V316',
'V317',
'V318',
'V319',
'V320',
'V321',
'V322',
'V323',
'V324',
'V325',
'V326',
'V327',
'V328',
'V329',
'V330',
'V331',
'V332',
'V333',
'V334',
'V335',
'V336',
'V337',
'V338',
'V339',
'id_01',
'id_02',
'id_03',
'id_04',
'id_05',
'id_06',
'id_07',
'id_08',
'id_09',
'id_10',
'id_11',
'id_13',
'id_14',
'id_17',
'id_18',
'id_19',
'id_20',
'id_21',
'id_22',
'id_24',
'id_25',
'id_26',
'id_32'
]<load_from_csv>
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
X_train = pd.read_csv('.. /input/data-cleaning-and-feature-engineering/train.csv', dtype=dict.fromkeys(float_cols, np.float32))
X_test = pd.read_csv('.. /input/data-cleaning-and-feature-engineering/test.csv', dtype=dict.fromkeys(float_cols, np.float32))
y_train = X_train['isFraud'].copy()
X_train.drop(columns=['isFraud'], inplace=True)
 print('Number of Training Examples = {}'.format(X_train.shape[0]))
print('Number of Test Examples = {}'.format(X_test.shape[0]))
print('Training Set Memory Usage = {:.2f} MB'.format(X_train.memory_usage().sum() / 1024**2))
print('Test Set Memory Usage = {:.2f} MB
'.format(X_test.memory_usage().sum() / 1024**2))
print('X_train Shape = {}'.format(X_train.shape))
print('y_train Shape = {}'.format(y_train.shape))
print('X_test Shape = {}
'.format(X_test.shape))<define_variables>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
drop_cols = ['V300','V309','V111','C3','V124','V106','V125','V315','V134','V102','V123','V316','V113',
'V136','V305','V110','V299','V289','V286','V318','V103','V304','V116','V29','V284','V293',
'V137','V295','V301','V104','V311','V115','V109','V119','V321','V114','V133','V122','V319',
'V105','V112','V118','V117','V121','V108','V135','V320','V303','V297','V120', 'TransactionID',
'TransactionDate', 'Minute', 'Hour', 'Day', 'DayOfWeek', 'Week', 'Month', 'card']
for df in [X_train, X_test]:
df.drop(columns=drop_cols, inplace=True )<categorify>
optR = OptimizedRounder()
optR.fit(oof_train, X_train_non_null['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train_non_null['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
object_cols = [col for col in X_train.columns if X_train[col].dtype == 'object']
le = LabelEncoder()
for df in [X_train, X_test]:
for col in object_cols:
df[col] = le.fit_transform(df[col].astype(str ).values )<init_hyperparams>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
lgb_param = {
'min_data_in_leaf': 106,
'num_leaves': 500,
'learning_rate': 0.008,
'min_child_weight': 0.03454472573214212,
'bagging_fraction': 0.4181193142567742,
'feature_fraction': 0.3797454081646243,
'reg_lambda': 0.6485237330340494,
'reg_alpha': 0.3899927210061127,
'max_depth': -1,
'objective': 'binary',
'seed': SEED,
'feature_fraction_seed': SEED,
'bagging_seed': SEED,
'drop_seed': SEED,
'data_random_seed': SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric':'auc',
}<split>
Counter(train_predictions )
%%time
N = 10
kf = KFold(n_splits=N)
 importance = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=X_train.columns)
 scores = []
y_pred = np.zeros(X_test.shape[0])
 oof = np.zeros(X_train.shape[0])
 for fold,(trn_idx, val_idx)in enumerate(kf.split(X_train, y_train), 1):
print('Fold {}'.format(fold))
trn_data = lgb.Dataset(X_train.iloc[trn_idx, :].values, label=y_train.iloc[trn_idx].values)
 val_data = lgb.Dataset(X_train.iloc[val_idx, :].values, label=y_train.iloc[val_idx].values)
 clf = lgb.train(lgb_param, trn_data, 10000, valid_sets=[trn_data, val_data], verbose_eval=500, early_stopping_rounds=500)
 predictions = clf.predict(X_train.iloc[val_idx, :].values)
 importance.iloc[:, fold - 1] = clf.feature_importance()
oof[val_idx] = predictions
score = roc_auc_score(y_train.iloc[val_idx].values, predictions)
 scores.append(score)
 print('Fold {} ROC AUC Score {}
'.format(fold, score))
y_pred += clf.predict(X_test)/ N
del trn_data, val_data, predictions
gc.collect()
print('Average ROC AUC Score {} [STD:{}]'.format(np.mean(scores), np.std(scores)) )<save_to_csv>
Counter(test_predictions )
submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID')
 submission['isFraud'] = y_pred
submission.to_csv('submission.csv')
 submission.head()<set_options>
submission = pd.DataFrame({'PetID': testIds.values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()
warnings.filterwarnings('ignore')
 sns.set_style('darkgrid')
 pd.options.display.float_format = '{:,.3f}'.format
print(os.listdir(".. /input"))<load_from_csv>
%%time
train_id = pd.read_csv('.. /input/train_identity.csv')
 train_trn = pd.read_csv('.. /input/train_transaction.csv')
 test_id = pd.read_csv('.. /input/test_identity.csv')
 test_trn = pd.read_csv('.. /input/test_transaction.csv' )<categorify>
<define_variables><EOS>