Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
warnings.filterwarnings("ignore" )
masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str)else 0)
 unique_img_ids = masks.groupby('ImageId' ).agg({'ships': 'sum'} ).reset_index()
unique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)
 unique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])
 unique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id:
os.stat(os.path.join(train_image_dir,
c_img_id)).st_size/1024)
 unique_img_ids = unique_img_ids[unique_img_ids['file_size_kb'] > 50]
unique_img_ids['file_size_kb'].hist()
masks.drop(['ships'], axis=1, inplace=True)
 unique_img_ids.sample(7 )<split>
traindf = pd.read_csv('.. /input/titanic/train.csv' ).set_index('PassengerId')
 testdf = pd.read_csv('.. /input/titanic/test.csv' ).set_index('PassengerId')
 submission = pd.read_csv('.. /input/titanic/gender_submission.csv' )
train_ids, valid_ids = train_test_split(balanced_train_df,
test_size = 0.3,
stratify = balanced_train_df['ships'])
 train_df = pd.merge(masks, train_ids)
 valid_df = pd.merge(masks, valid_ids)
 print(train_df.shape[0], 'training masks')
 print(valid_df.shape[0], 'validation masks' )<categorify>
df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - \
df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs')
 df['Embarked'] = df['Embarked'].fillna('S')
 df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare)
 df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
cols_to_drop = ['Name','Ticket','Cabin', 'IsWomanOrBoy', 'WomanOrBoyCount', 'FamilySurvivedCount']
df = df.drop(cols_to_drop, axis=1)
 df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.Alone = df.Alone.fillna(0)
 target = df.Survived.loc[traindf.index]
df = df.drop(['Survived'], axis=1)
 train, test = df.loc[traindf.index], df.loc[testdf.index]
def make_image_gen(in_df, batch_size = BATCH_SIZE):
all_batches = list(in_df.groupby('ImageId'))
out_rgb = []
out_mask = []
while True:
np.random.shuffle(all_batches)
 for c_img_id, c_masks in all_batches:
rgb_path = os.path.join(train_image_dir, c_img_id)
 c_img = imread(rgb_path)
 c_mask = np.expand_dims(masks_as_image(c_masks['EncodedPixels'].values), -1)
 if IMG_SCALING is not None:
c_img = c_img[::IMG_SCALING[0], ::IMG_SCALING[1]]
c_mask = c_mask[::IMG_SCALING[0], ::IMG_SCALING[1]]
out_rgb += [c_img]
out_mask += [c_mask]
if len(out_rgb)>=batch_size:
yield np.stack(out_rgb, 0)/255.0, np.stack(out_mask, 0)
 out_rgb, out_mask=[], []<train_model>
pd.set_option('max_columns',100 )
train_gen = make_image_gen(train_df)
 train_x, train_y = next(train_gen)
 print('x', train_x.shape, train_x.min() , train_x.max())
 print('y', train_y.shape, train_y.min() , train_y.max() )<train_model>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = train.columns.values.tolist()
for col in features:
if train[col].dtype in numerics: continue
categorical_columns.append(col)
 for col in categorical_columns:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
%%time
valid_x, valid_y = next(make_image_gen(valid_df, VALID_IMG_COUNT))
print(valid_x.shape, valid_y.shape )<set_options>
train = reduce_mem_usage(train )
dg_args = dict(featurewise_center = False,
samplewise_center = False,
rotation_range = 45,
width_shift_range = 0.1,
height_shift_range = 0.1,
shear_range = 0.01,
zoom_range = [0.9, 1.25],
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'reflect',
data_format = 'channels_last')
 if AUGMENT_BRIGHTNESS:
dg_args[' brightness_range'] = [0.5, 1.5]
image_gen = ImageDataGenerator(**dg_args)
 if AUGMENT_BRIGHTNESS:
dg_args.pop('brightness_range')
 label_gen = ImageDataGenerator(**dg_args)
 def create_aug_gen(in_gen, seed = None):
np.random.seed(seed if seed is not None else np.random.choice(range(9999)))
 for in_x, in_y in in_gen:
seed = np.random.choice(range(9999))
g_x = image_gen.flow(255*in_x,
batch_size = in_x.shape[0],
seed = seed,
shuffle=True)
 g_y = label_gen.flow(in_y,
batch_size = in_x.shape[0],
seed = seed,
shuffle=True)
 yield next(g_x)/255.0, next(g_y )<set_options>
test = reduce_mem_usage(test )
gc.collect()<define_search_model>
X = train
z = target
def upsample_conv(filters, kernel_size, strides, padding):
return layers.Conv2DTranspose(filters, kernel_size, strides=strides, padding=padding)
 def upsample_simple(filters, kernel_size, strides, padding):
return layers.UpSampling2D(strides)
 if UPSAMPLE_MODE=='DECONV':
upsample=upsample_conv
else:
upsample=upsample_simple
input_img = layers.Input(t_x.shape[1:], name = 'RGB_Input')
 pp_in_layer = input_img
if NET_SCALING is not None:
pp_in_layer = layers.AvgPool2D(NET_SCALING )(pp_in_layer)
 pp_in_layer = layers.GaussianNoise(GAUSSIAN_NOISE )(pp_in_layer)
 pp_in_layer = layers.BatchNormalization()(pp_in_layer)
 c1 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(pp_in_layer)
 c1 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(c1)
 p1 = layers.MaxPooling2D(( 2, 2))(c1)
 c2 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(p1)
 c2 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(c2)
 p2 = layers.MaxPooling2D(( 2, 2))(c2)
 c3 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(p2)
 c3 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(c3)
 p3 = layers.MaxPooling2D(( 2, 2))(c3)
 c4 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(p3)
 c4 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(c4)
 p4 = layers.MaxPooling2D(pool_size=(2, 2))(c4)
 c5 = layers.Conv2D(128,(3, 3), activation='relu', padding='same' )(p4)
 c5 = layers.Conv2D(128,(3, 3), activation='relu', padding='same' )(c5)
 u6 = upsample(64,(2, 2), strides=(2, 2), padding='same' )(c5)
 u6 = layers.concatenate([u6, c4])
 c6 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(u6)
 c6 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(c6)
 u7 = upsample(32,(2, 2), strides=(2, 2), padding='same' )(c6)
 u7 = layers.concatenate([u7, c3])
 c7 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(u7)
 c7 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(c7)
 u8 = upsample(16,(2, 2), strides=(2, 2), padding='same' )(c7)
 u8 = layers.concatenate([u8, c2])
 c8 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(u8)
 c8 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(c8)
 u9 = upsample(8,(2, 2), strides=(2, 2), padding='same' )(c8)
 u9 = layers.concatenate([u9, c1], axis=3)
 c9 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(u9)
 c9 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(c9)
 d = layers.Conv2D(1,(1, 1), activation='sigmoid' )(c9)
 if NET_SCALING is not None:
d = layers.UpSampling2D(NET_SCALING )(d)
 seg_model = models.Model(inputs=[input_img], outputs=[d])
 seg_model.summary()<compute_test_metric>
Xtrain, Xval, Ztrain, Zval = train_test_split(X, z, test_size=0.2, random_state=0)
 train_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)
 valid_set = lgbm.Dataset(Xval, Zval, silent=False )
def IoU(y_true, y_pred, eps=1e-6):
intersection = K.sum(y_true * y_pred, axis=[1,2,3])
 union = K.sum(y_true, axis=[1,2,3])+ K.sum(y_pred, axis=[1,2,3])- intersection
return K.mean(( intersection + eps)/(union + eps), axis=0)
 def zero_IoU(y_true, y_pred):
return IoU(1-y_true, 1-y_pred)
 def agg_loss(in_gt, in_pred):
return -1e-2 * zero_IoU(in_gt, in_pred)- IoU(in_gt, in_pred)
 seg_model.compile(optimizer=Adam(1e-3, decay=1e-6), loss=agg_loss, metrics=[IoU, zero_IoU, 'binary_accuracy'] )<choose_model_class>
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 5000 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
}
modelL = lgbm.train(params, train_set = train_set, num_boost_round=1000,
early_stopping_rounds=50, verbose_eval=10, valid_sets=valid_set )
weight_path="{}_weights.best.hdf5".format('seg_model')
 checkpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=1, verbose=1, mode='min',
min_delta=0.0001, cooldown=2, min_lr=1e-7)
 early = EarlyStopping(monitor="val_loss", mode="min", verbose=2,
patience=15)
 callbacks_list = [checkpoint, early, reduceLROnPlat]<train_model>
feature_score = pd.DataFrame(train.columns, columns = ['feature'])
 feature_score['score_lgb'] = modelL.feature_importance()
step_count = min(MAX_TRAIN_STEPS, train_df.shape[0]//BATCH_SIZE)
 aug_gen = create_aug_gen(make_image_gen(train_df))
loss_history = [seg_model.fit_generator(aug_gen,
steps_per_epoch=step_count,
epochs=10,
validation_data=(valid_x, valid_y),
callbacks=callbacks_list,
workers=1,
max_queue_size = 20,use_multiprocessing=True,verbose=1)]<load_pretrained>
y_train_lgb = modelL.predict(train, num_iteration=modelL.best_iteration ).astype('int')
 y_preds_lgb = modelL.predict(test, num_iteration=modelL.best_iteration )
seg_model.load_weights(weight_path)
 seg_model.save('seg_model.h5' )<predict_on_test>
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 data_train = xgb.DMatrix(train)
 data_test = xgb.DMatrix(test)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]
pred_y = seg_model.predict(valid_x)
 print(pred_y.shape, pred_y.min() , pred_y.max() , pred_y.mean() )<choose_model_class>
parms = {'max_depth':8,
'objective':'reg:logistic',
'eta' :0.3,
'subsample':0.8,
'lambda ' :4,
'colsample_bytree ':0.9,
'colsample_bylevel':1,
'min_child_weight': 10}
modelx = xgb.train(parms, data_tr, num_boost_round=200, evals = evallist,
early_stopping_rounds=30, maximize=False,
verbose_eval=10)
 print('score = %1.5f, n_boost_round =%d.'%(modelx.best_score,modelx.best_iteration))
if IMG_SCALING is not None:
fullres_model = models.Sequential()
fullres_model.add(layers.AvgPool2D(IMG_SCALING, input_shape =(None, None, 3)))
 fullres_model.add(seg_model)
 fullres_model.add(layers.UpSampling2D(IMG_SCALING))
else:
fullres_model = seg_model
fullres_model.save('fullres_model.h5' )<predict_on_test>
feature_score['score_xgb'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))
feature_score
def predict(img, path=test_image_dir):
c_img = imread(os.path.join(path, c_img_name))
c_img = np.expand_dims(c_img, 0)/255.0
cur_seg = fullres_model.predict(c_img)[0]
cur_seg = binary_opening(cur_seg>1e3, np.expand_dims(disk(2), -1))
return cur_seg, c_img
def pred_encode(img):
cur_seg, _ = predict(img)
 cur_rles = rle_encode(cur_seg)
 return [img, cur_rles if len(cur_rles)> 0 else None]
samples = train_df.groupby('grouped_ship_count' ).apply(lambda x: x.sample(1))
fig, m_axs = plt.subplots(samples.shape[0], 3, figsize =(11, samples.shape[0]*4))
[c_ax.axis('off')for c_ax in m_axs.flatten() ]
for(ax1, ax2, ax3), c_img_name in zip(m_axs, samples.ImageId.values):
first_seg, first_img = predict(c_img_name, train_image_dir)
 ax1.imshow(first_img[0])
 ax1.set_title('Image')
 ax2.imshow(first_seg[:, :, 0])
 ax2.set_title('Prediction')
 ground_truth = masks_as_color(masks.query('ImageId=="{}"'.format(c_img_name)) ['EncodedPixels'])
 ax3.imshow(ground_truth)
 ax3.set_title('Ground Truth')
 fig.savefig('predictions.png' )<define_variables>
y_train_xgb = modelx.predict(data_train ).astype('int')
 y_preds_xgb = modelx.predict(data_test )
test_paths = np.array(os.listdir(test_image_dir))
print(len(test_paths), 'test images found' )<define_variables>
Scaler_train = preprocessing.MinMaxScaler()
train = pd.DataFrame(
 Scaler_train.fit_transform(train),
columns=train.columns,
index=train.index
)
%%time
out_pred_rows = []
for c_img_name in tqdm_notebook(test_paths[:30000]):
out_pred_rows += [pred_encode(c_img_name)]<prepare_output>
test = pd.DataFrame(
 Scaler_train.fit_transform(test),
columns=test.columns,
index=test.index
)
sub = pd.DataFrame(out_pred_rows)
 sub.columns = ['ImageId', 'EncodedPixels']
sub = sub[sub.EncodedPixels.notnull() ]
sub.head()<save_to_csv>
logreg = LogisticRegression()
logreg.fit(train, target)
 coeff_logreg = pd.DataFrame(train.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 coeff_logreg.sort_values(by='score_logreg', ascending=False )
sub1 = pd.read_csv('.. /input/sample_submission.csv')
 sub1 = pd.DataFrame(np.setdiff1d(sub1['ImageId'].unique() , sub['ImageId'].unique() , assume_unique=True), columns=['ImageId'])
 sub1['EncodedPixels'] = None
print(len(sub1), len(sub))
sub = pd.concat([sub, sub1])
 print(len(sub))
sub.to_csv('submission.csv', index=False)
 sub.head()<train_model>
coeff_logreg["score_logreg"] = coeff_logreg["score_logreg"].abs()
feature_score = pd.merge(feature_score, coeff_logreg, on='feature' )
<set_options>
eli5.show_weights(logreg )
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128
plt.rcParams['figure.figsize'] =(12, 9 )<load_from_csv>
y_train_logreg = logreg.predict(train ).astype('int')
 y_preds_logreg = logreg.predict(test )
train = pd.read_csv('.. /input/train/train.csv')
 print('train shape:', train.shape)
 test = pd.read_csv('.. /input/test/test.csv')
 print('test shape:', test.shape)
 sample_submission = pd.read_csv('.. /input/test/sample_submission.csv')
 labels_breed = pd.read_csv('.. /input/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/color_labels.csv')
 labels_color = pd.read_csv('.. /input/state_labels.csv')
 train_image_files = sorted(glob.glob('.. /input/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )<count_unique_values>
linreg = LinearRegression()
linreg.fit(train, target)
 coeff_linreg = pd.DataFrame(train.columns.delete(0))
coeff_linreg.columns = ['feature']
coeff_linreg["score_linreg"] = pd.Series(linreg.coef_)
 coeff_linreg.sort_values(by='score_linreg', ascending=False )
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print()
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )<load_pretrained>
eli5.show_weights(linreg )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<feature_engineering>
coeff_linreg["score_linreg"] = coeff_linreg["score_linreg"].abs()
aggregates = ['mean', 'sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
feature_score = pd.merge(feature_score, coeff_linreg, on='feature')
 feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
y_train_linreg = linreg.predict(train ).astype('int')
 y_preds_linreg = linreg.predict(test )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<categorify>
feature_score.sort_values('mean', ascending=False )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 {}'.format(np.sum(pd.isnull(X))))
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))<groupby>
w_lgb = 0.48
w_xgb = 0.48
w_logreg = 0.03
w_linreg = 1 - w_lgb - w_xgb - w_logreg
w_linreg
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )<feature_engineering>
feature_score.sort_values('merging', ascending=False )
n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1)
 X_temp = X_temp.drop(to_drop_columns, axis=1)
 print('X shape: {}'.format(X_temp.shape))<define_variables>
y_preds = w_lgb*y_preds_lgb + w_xgb*y_preds_xgb + w_logreg*y_preds_logreg + w_linreg*y_preds_linreg
X_temp_column_types = X_temp.dtypes
X_temp_int_cols = X_temp_column_types[X_temp_column_types == 'int']
X_temp_float_cols = X_temp_column_types[X_temp_column_types == 'float']
X_temp_cat_cols = X_temp_column_types[X_temp_column_types == 'object']
print('\tinteger columns:
{}'.format(X_temp_int_cols))
print('
\tfloat columns:
{}'.format(X_temp_float_cols))
print('
\tto encode categorical columns:
{}'.format(X_temp_cat_cols))<import_modules>
submission['Survived'] = [1 if x>0.5 else 0 for x in y_preds]
submission.head()
from sklearn.datasets import make_hastie_10_2
from sklearn.ensemble import GradientBoostingClassifier
import matplotlib.pyplot as plt<split>
submission.to_csv('submission.csv', index=False )
<prepare_x_and_y><EOS> 
submission.to_csv('submission.csv', index=False )