Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<define_variables>
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic')
 def warn(*args, **kwargs):
pass
warnings.warn = warn
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128
cutoff_lo = 0.3
cutoff_hi = 0.7<save_to_csv>
plt.rcParams['figure.figsize'] =(12, 9)
test_df['Label'] = test_df['mean']
test_df[['Id', 'Label']].to_csv('stack_mean.csv', index=False, float_format='%.6f')
 test_df['Label'] = test_df['median']
test_df[['Id', 'Label']].to_csv('stack_median.csv', index=False, float_format='%.6f' )<save_to_csv>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
test_df['Label'] = np.where(np.all(test_df.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(test_df.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, test_df['median']))
test_df[['Id', 'Label']].to_csv('stack_pushout_median.csv', index=False, float_format='%.6f' )<feature_engineering>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
test_df['Label'] = np.where(np.all(test_df.iloc[:,1:ncol] > cutoff_lo, axis=1),
test_df['max'],
np.where(np.all(test_df.iloc[:,1:ncol] < cutoff_hi, axis=1),
test_df['min'],
test_df['mean']))
test_df[['Id', 'Label']].to_csv('stack_minmax_mean.csv', index=False, float_format='%.6f' )<feature_engineering>
def maek_features(src_df):
rescuer_count = src_df.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_CNT']
src_df = src_df.merge(rescuer_count, how='left', on='RescuerID')
 return src_df
test_df['Label'] = np.where(np.all(test_df.iloc[:,1:ncol] > cutoff_lo, axis=1),
test_df['max'],
np.where(np.all(test_df.iloc[:,1:ncol] < cutoff_hi, axis=1),
test_df['min'],
test_df['median']))
test_df[['Id', 'Label']].to_csv('stack_minmax_median.csv', index=False, float_format='%.6f' )<set_options>
train = maek_features(train)
 test = maek_features(test )
%matplotlib inline 
<set_options>
train_img = pd.read_csv(".. /input/pretrainednn/train_img_features.csv")
 test_img = pd.read_csv(".. /input/pretrainednn/test_img_features.csv")
 train_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 test_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 train = pd.concat([train, train_img], axis=1)
 test = pd.concat([test, test_img], axis=1 )
%matplotlib inline 
<load_from_csv>
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f )
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))<load_from_csv>
cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))<feature_engineering>
breed_id = {}
for id,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[id] = name
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )<categorify>
breed_names_1 = [i for i in cat_ratings.keys() ]
breed_names_2 = [i for i in dog_ratings.keys() ]
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))<categorify>
for id in train['Breed1']:
if id in breed_id.keys() :
name = breed_id[id]
if name in breed_names_1:
for key in cat_ratings[name].keys() :
train[key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
train[key] = dog_ratings[name][key]
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))<categorify>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )
m = phone.phone_brand.str.cat(phone.device_model)
 modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))<create_dataframe>
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 deviceapps.head()<prepare_x_and_y>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))<categorify>
aggregates = ['mean', 'sum', 'var']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )<categorify>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )<merge>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()<correct_missing_values>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))<concatenate>
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))<concatenate>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))<categorify>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
targetencoder = LabelEncoder().fit(gatrain.gender)
 y = targetencoder.transform(gatrain.gender)
 nclasses = len(targetencoder.classes_ )<categorify>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
clf = LogisticRegression(C=0.08)
 clf.fit(Xtrain[70001:], y[70001:])
 pred = pd.DataFrame(clf.predict_proba(Xtrain[70001:]), index=gatrain.iloc[70001:].index, columns=targetencoder.classes_)
 pred.head()<save_to_csv>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )
pred.to_csv('test_gender.csv',index=True )<import_modules>
n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPool2D, Flatten, BatchNormalization, Dropout
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools<load_from_csv>
np.sum(pd.isnull(X_train))
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<prepare_x_and_y>
np.sum(pd.isnull(X_test))
X_train = train.iloc[:,1:]
y_train = train.iloc[:,0]<categorify>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
X_train = X_train.values.reshape(-1, 28, 28, 1)/255.
 test = test.values.reshape(-1, 28, 28, 1)/255.
 y_train = to_categorical(y_train, 10 )<split>
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 10000
n_splits = 5
random_seed = 0
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=random_seed )<choose_model_class>
X_train = X_train.drop('img_Unnamed: 0',axis=1)
 X_test = X_test.drop('img_Unnamed: 0',axis=1 )
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1
)<choose_model_class>
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train_lgb = np.zeros(( X_train.shape[0]))
oof_test_lgb = np.zeros(( X_test.shape[0], n_splits))
qwk_scores = []
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(val_pred, y_val)
 coefficients = optR.coefficients()
pred_val_y_k = optR.predict(val_pred, coefficients)
 print("Valid Counts = ", Counter(y_val))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(y_val, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train_lgb[valid_index] = val_pred
oof_test_lgb[:, i] = test_pred
i += 1
print('{} cv mean QWK score : {}'.format('LGBM', np.mean(qwk_scores)) )
model = Sequential()
model.add(Conv2D(32,(5,5), padding='same', input_shape=X_train.shape[1:], activation='relu'))
model.add(Conv2D(32,(5,5), padding='same', activation='relu'))
model.add(MaxPool2D(2,2))
model.add(Conv2D(64,(3,3), padding='same', activation='relu'))
model.add(Conv2D(64,(3,3), padding='same', activation='relu'))
model.add(MaxPool2D(2,2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary()<choose_model_class>
importance_type= "split"
idx_sort = np.argsort(model.feature_importance(importance_type=importance_type)) [::-1]
names_sorted = np.array(model.feature_name())[idx_sort]
imports_sorted = model.feature_importance(importance_type=importance_type)[idx_sort]
for n, im in zip(names_sorted, imports_sorted):
print(n, im )
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<train_model>
optR = OptimizedRounder()
optR.fit(oof_train_lgb, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train_lgb, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk )
EPOCHS = 30
BATCH_SIZE = 20
callback_list = [
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1),
EarlyStopping(monitor='val_loss', min_delta=0.0005, patience=4)
 ]
history = model.fit(datagen.flow(X_train, y_train, batch_size=BATCH_SIZE),
epochs=EPOCHS,
callbacks=callback_list,
validation_data=(X_val, y_val),
steps_per_epoch=X_train.shape[0] // BATCH_SIZE )<save_to_csv>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions_lgb = optR.predict(oof_train_lgb, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions_lgb)))
 test_predictions_lgb = optR.predict(oof_test_lgb.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions_lgb)) )
results = model.predict(test)
 results = np.argmax(results, axis=1)
 results = pd.Series(results, name='Label')
 submission = pd.concat([pd.Series(range(1,28001), name='ImageID'), results], axis=1)
 submission.to_csv('submission.csv', index=False )<set_options>
print("True Distribution:")
 print(pd.value_counts(X_train['AdoptionSpeed'], normalize=True ).sort_index())
 print("
Train Predicted Distribution:")
 print(pd.value_counts(train_predictions_lgb, normalize=True ).sort_index())
 print("
Test Predicted Distribution:")
 print(pd.value_counts(test_predictions_lgb, normalize=True ).sort_index() )
%%script false --no-raise-error
print(device_lib.list_local_devices())
<import_modules>
df = pd.concat([X_train, X_test], axis=0)
 df.head(2 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
from sklearn.model_selection import train_test_split, StratifiedKFold
from keras import backend as K
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import to_categorical
from keras.models import Sequential, load_model
from keras.layers import Conv2D, BatchNormalization, Dropout, Flatten, Dense
from keras.callbacks import LearningRateScheduler
import tensorflow as tf 
<set_options>
df_ = pd.concat([train,test],axis=0 )
gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
tf.config.experimental.set_visible_devices(gpus[0], 'GPU')
 for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e )<load_from_csv>
word_vec_size = 300
max_words = 100
max_word_features = 25000
def transform_text(text, tokenizer):
tokenizer.fit_on_texts(text)
 text_emb = tokenizer.texts_to_sequences(text)
 text_emb = sequence.pad_sequences(text_emb, maxlen=max_words)
 return text_emb
desc_tokenizer = text.Tokenizer(num_words=max_word_features)
 desc_embs = transform_text(df_["Description"].astype(str), desc_tokenizer )
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer//test.csv' )<data_type_conversions>
text_mode = "fasttext"
if text_mode == "fasttext":
embedding_file = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(embedding_file))
word_index = desc_tokenizer.word_index
print('Word index len:', len(word_index))
vocabulary_size = min(max_word_features, len(word_index)) + 1
text_embs = np.zeros(( vocabulary_size, word_vec_size))
for word, i in word_index.items() :
if i >= max_word_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: text_embs[i] = embedding_vector
del(embeddings_index)
 elif text_mode == "word2vec":
embedding_file = ".. /input/word2vec-google/GoogleNews-vectors-negative300.bin"
print("Loading word vectors...")
 word_vectors = KeyedVectors.load_word2vec_format(embedding_file, binary=True)
 print("Matching word vectors...")
 EMBEDDING_DIM=300
word_index = desc_tokenizer.word_index
vocabulary_size=min(len(word_index)+1,max_word_features)
 text_embs = np.zeros(( vocabulary_size, EMBEDDING_DIM))
for word, i in word_index.items() :
if i>=max_word_features:
continue
try:
embedding_vector = word_vectors[word]
text_embs[i] = embedding_vector
except KeyError:
text_embs[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)
 del(word_vectors )
y = train.pop('label')
 train_y = to_categorical(y)
 train_X = train/255
test_X = test/255
train_X = train_X.to_numpy()
test_X = test_X.to_numpy()<split>
np.sum(pd.isnull(df))
sub_test_X, sub_train_X, sub_test_y, sub_train_y = train_test_split(train_X, train_y,
train_size=0.2, stratify=train_y )<choose_model_class>
cat_vars = ["Type", "Breed1", "Breed2", "Color1", "Color2", "Color3", "Gender", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "Health", "State"]
cont_vars = ["Fee", "PhotoAmt", "VideoAmt", "Age", "Quantity",'RescuerID_CNT']
img_gen = ImageDataGenerator(rotation_range = 12, width_shift_range=.12, height_shift_range=.12,
zoom_range=.12 )<choose_model_class>
def preproc(df):
global cont_vars
for var in cat_vars:
df[var] = LabelEncoder().fit_transform(df[var])
 for var in cont_vars:
df[var] = MinMaxScaler().fit_transform(df[var].values.reshape(-1,1))
return df
def build_model(save = False):
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation = 'relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation ='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 return model
def train_model(model, train_X, train_y, save = False):
sub_test_X, sub_train_X, sub_test_y, sub_train_y = train_test_split(train_X, train_y,
train_size=0.2, stratify=train_y)
 annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 start = time.clock()
model.fit_generator(img_gen.flow(sub_train_X, sub_train_y), epochs=10,
steps_per_epoch=sub_train_X.shape[0]//64,
validation_data =(sub_test_X,sub_test_y), callbacks=[annealer])
 print(f'Overall it takes {time.clock() -start} sec')
 if save ==True:
model.save('/kaggle/input/trained-model/model.h5')
 def load_my_model() :
re_model = load_model('/kaggle/input/trained-model/first_model.h5')
 return re_model 
<train_model>
df_scaled = preproc(df)
 train_df = df_scaled[:len(train)]
test_df = df_scaled[len(train):]
len(train_df), len(test_df )
num_models = 3
models = [0]*num_models
for i in range(num_models):
models[i] = build_model()
train_model(models[i],train_X, train_y)
<predict_on_test>
def get_keras_data(df, description_embeds):
X = {var: df[var].values for var in cont_vars+cat_vars}
X["description"] = description_embeds
for i in range(256): X[f"img_{i}"] = df[f"img_{i}"]
return X
prediction = np.zeros(( test_X.shape[0],10))
for i in range(len(models)) :
prediction += models[i].predict(test_X )<prepare_output>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class QWKEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.history = []
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, batch_size=1000, verbose=0)
 y_pred = eval_predict(self.y_val, y_pred)
 score = quadratic_weighted_kappa(self.y_val, y_pred)
 print("QWK - epoch: %d - score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history): self.model.save('checkpoint.h5')
 def eval_predict(y=[], y_pred=[], coeffs=None, ret_coeffs=False):
optR = OptimizedRounder()
if not coeffs:
optR.fit(y_pred.reshape(-1,), y)
 coeffs = optR.coefficients()
if ret_coeffs: return optR.coefficients()
return optR.predict(y_pred, coeffs ).reshape(-1, )
predict = np.argmax(prediction, axis =1)
 predict = np.vstack(( np.arange(predict.shape[0])+1, predict)).T<create_dataframe>
def rmse(y, y_pred):
return K.sqrt(K.mean(K.square(y-y_pred), axis=-1))
def get_model(emb_n=10, dout=.25, batch_size=1000):
inps = []
embs = []
nums = []
for var in cat_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 embs.append(( Embedding(df[var].max() +1, emb_n )(inp)))
 for var in cont_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 nums.append(( inp))
desc_inp = Input(shape=(max_words,), name="description")
 inps.append(desc_inp)
 emb_desc = Embedding(vocabulary_size, word_vec_size, weights=[text_embs] )(desc_inp)
 emb_desc = SpatialDropout1D (.4 )(emb_desc)
 emb_desc = Reshape(( max_words, word_vec_size, 1))(emb_desc)
 filter_sizes=[1,3,1]
convs = []
for filter_size in filter_sizes:
conv = Conv2D(32, kernel_size=(filter_size, word_vec_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool2D(pool_size=(max_words-filter_size+1, 1))(conv))
img_fts = []
for i in range(256):
inp = Input(shape=[1], name=f"img_{i}")
 inps.append(inp)
 img_fts.append(( inp))
img_fts = concatenate(img_fts)
 img_fts = Dropout (.5 )(img_fts)
 img_fts = BatchNormalization()(img_fts)
 img_fts = Dense(64, activation="relu" )(img_fts)
 convs = concatenate(convs)
 convs = Flatten()(convs)
 convs = BatchNormalization()(convs)
 embs = Flatten()(concatenate(embs))
embs = Dropout(dout )(Dense(64, activation="relu", kernel_initializer="he_normal" )(embs))
nums = concatenate(nums)
 nums = Dense(32, activation="relu" )(nums)
 nums = Dropout(0.5 )(nums)
 x = concatenate([embs, nums, convs, img_fts])
 x = BatchNormalization()(x)
 dense_n = [256, 64]
for n in dense_n:
x = BatchNormalization()(x)
 x = Dense(n, activation="relu", kernel_initializer="he_normal" )(x)
 x = BatchNormalization()(x)
 x = Dropout (.2 )(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inps, outputs=out)
 opt = Adam()
model.compile(optimizer=opt, loss='mse',)
 return model
submission = pd.DataFrame(data=predict, columns=['imageid', 'label'] )<save_to_csv>
nfolds=5
folds = StratifiedKFold(n_splits=nfolds,shuffle=True, random_state=15)
 avg_train_kappa = 0
avg_valid_kappa = 0
batch_size=1000
coeffs=None
x_test = get_keras_data(test_df, desc_embs[len(train_df):])
 adoptions_keras = np.zeros(( len(test_df),))
oof_train_keras = np.zeros(( train_df.shape[0]))
i =0
for train_idx, valid_idx in folds.split(train_df[cat_vars+cont_vars], train_df["AdoptionSpeed"]):
x_train = get_keras_data(train_df.iloc[train_idx], desc_embs[train_idx])
 x_valid = get_keras_data(train_df.iloc[valid_idx], desc_embs[valid_idx])
 y_train, y_valid = train_df["AdoptionSpeed"][train_idx].values, train_df["AdoptionSpeed"][valid_idx].values
model = get_model()
clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(train_df)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(validation_data=(x_valid, y_valid), interval=1)
 history = model.fit(x_train, y_train, batch_size=batch_size, validation_data=(x_valid, y_valid),
epochs=20, callbacks=[clr_tri, qwk_eval])
 model.load_weights('checkpoint.h5')
 y_train_pred = eval_predict(y_train, model.predict(x_train, batch_size=1000))
y_valid_pred = eval_predict(y_valid, model.predict(x_valid, batch_size=1000))
avg_train_kappa += quadratic_weighted_kappa(y_train_pred, y_train)
 avg_valid_kappa += quadratic_weighted_kappa(y_valid_pred, y_valid)
 coeffs = eval_predict(y_valid, model.predict(x_valid, batch_size=1000), ret_coeffs=True)
 adoptions_keras += model.predict(x_test, batch_size=batch_size ).reshape(-1,)
 oof_train_keras[valid_idx] = y_valid_pred
print("
avg train kappa:", avg_train_kappa/nfolds,)
 print("
avg valid kappa:", avg_valid_kappa/nfolds, )
submission.to_csv('submit.csv',index=False )<set_options>
Counter(X_train['AdoptionSpeed'] )
%matplotlib inline 
<set_options>
Counter(X_train['AdoptionSpeed'] )
print("Tensorflow version " + tf.__version__)
 np.random.seed(42 )<set_options>
coeffs[0] = 1.645
coeffs[1] = 2.115
coeffs[3] = 2.84
print('True Counter',Counter(X_train['AdoptionSpeed']))
train_pred_keras = eval_predict(y_pred=oof_train_keras, coeffs=list(coeffs)).astype(int)
 print('Train pred Counter',Counter(train_pred_keras))
test_pred_keras = eval_predict(y_pred=adoptions_keras/nfolds, coeffs=list(coeffs)).astype(int)
 print('Test pred Counter',Counter(test_pred_keras))
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
train_stack = np.vstack([train_pred_keras, train_predictions_lgb] ).transpose()
test_stack = np.vstack([test_pred_keras, test_predictions_lgb] ).transpose()
train_data=pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_data=pd.read_csv(".. /input/digit-recognizer/test.csv" )<prepare_x_and_y>
from sklearn.linear_model import Ridge
Y_train=train_data['label']
X_train=train_data.drop('label', axis=1 )<prepare_x_and_y>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(train_stack.shape[0])
 predictions = np.zeros(test_stack.shape[0])
 qwk_scores = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, X_train['AdoptionSpeed'].values)) :
trn_data, trn_y = train_stack[trn_idx], X_train['AdoptionSpeed'].iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], X_train['AdoptionSpeed'].iloc[val_idx].values
clf = Ridge(alpha=1)
 clf.fit(trn_data, trn_y)
 val_pred = clf.predict(val_data)
 pred_val_y_k = eval_predict(y_pred=val_pred, coeffs=list(coeffs)).astype(int)
 print("Valid Counts = ", Counter(val_y))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(val_y, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 oof[val_idx] = clf.predict(val_data)
 predictions += clf.predict(test_stack)/ folds.n_splits
print('{} cv mean QWK score : {}'.format('Stacked_model_ridge', np.mean(qwk_scores)) )
( x_train0, y_train0),(x_test0, y_test0)= mnist.load_data()
x_train1 = np.concatenate([x_train0, x_test0], axis=0)
 y_train1 = np.concatenate([y_train0, y_test0], axis=0)
 X_train_keras = x_train1.reshape(-1, 28*28)
 Y_train_keras = y_train1<concatenate>
pred_final = eval_predict(y_pred=predictions, coeffs=list(coeffs)).astype(int )
X_train = np.concatenate(( X_train.values, X_train_keras))
Y_train = np.concatenate(( Y_train, Y_train_keras))<count_unique_values>
Counter(pred_final )
unique, counts = np.unique(Y_train, return_counts=True)
 dict(zip(unique, counts))<data_type_conversions>
Counter(test_pred_keras )
X_train = X_train.astype('float32')
 Y_train = Y_train.astype('float32')
 test_data=test_data.astype('float32')
 X_train = X_train / 255.0
test_data = test_data / 255.0<categorify>
Counter(test_predictions_lgb )
<split><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': pred_final.astype(np.int32)})
 submission.head()
submission.to_csv('submission.csv', index=False )