Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<find_best_params>
pd.set_option('max_columns', 140)
 pd.set_option('max_colwidth', 5000)
 pd.set_option('display.max_rows', 140)
 %matplotlib inline
plt.rcParams['figure.figsize'] =(12,8)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
val_prob = functions.sigmoid(val_pred ).data
best_th, best_fbeta = search_best_threshold(
 val_prob, val_label, eval_func=average_fbeta_score, search_range=[0.1, 0.9], interval=0.01)
 print(best_th, best_fbeta )<predict_on_test>
train = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 codebook = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/codebook.csv')
 df = train.append(test)
 train.shape,test.shape, df.shape
test_iter = iterators.MultiprocessIterator(
 test_dataset, settings["batch_size"], repeat=False, shuffle=False)
 test_pred, _ = predict(trained_model, test_iter, gpu_device=0 )<predict_on_test>
print(codebook[codebook['Variable name']=='rez_esc'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='v18q1'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='v2a1'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='meaneduc'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='SQBmeaned'] )
test_prob = functions.sigmoid(test_pred ).data
test_pred_ids = make_pred_ids(test_prob, th=best_th )<save_to_csv>
df.v18q1.value_counts()
test_df.to_csv("submission.csv", index=False )<init_hyperparams>
df.rez_esc.value_counts()
WORKERS = 2
CHANNEL = 3
NUM_CLASSES = 1103
warnings.filterwarnings("ignore")
 SIZE = 224
RESNET_VER = 2
epochs = 27
batch_size = 88
SPLIT_RATIO = 0.1
POST_PROCESS = False
LR = 5e-4
LR_FACTOR = 0.2
PATIENCE=2
LR_WARM = 1e-3
gamma = 0.0
checkpoint_file = '.. /working/resnet50_focal.h5'<load_from_csv>
df.v2a1.value_counts()
path_to_train = '.. /input/imet-2019-fgvc6/train/'
data = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_dataset_info = []
for name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')) :
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 train_dataset_info = np.array(train_dataset_info )<load_from_csv>
df.meaneduc.value_counts()
DATA_ROOT = '.. /input/imet-2019-fgvc6/'
def make_folds(n_folds: int)-> pd.DataFrame:
df = pd.read_csv(DATA_ROOT+ 'train.csv')
 cls_counts = Counter(cls for classes in df['attribute_ids'].str.split()
for cls in classes)
 fold_cls_counts = defaultdict(int)
 folds = [-1] * len(df)
 for item in tqdm(df.sample(frac=1, random_state=42 ).itertuples() ,
total=len(df)) :
cls = min(item.attribute_ids.split() , key=lambda cls: cls_counts[cls])
 fold_counts = [(f, fold_cls_counts[f, cls])for f in range(n_folds)]
min_count = min([count for _, count in fold_counts])
 random.seed(item.Index)
 fold = random.choice([f for f, count in fold_counts
if count == min_count])
 folds[item.Index] = fold
for cls in item.attribute_ids.split() :
fold_cls_counts[fold, cls] += 1
df['fold'] = folds
return df<compute_test_metric>
df.SQBmeaned.value_counts()
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = K.clip(pt, epsilon, 1-epsilon)
 CE = -K.log(pt)
 FL = K.pow(1-pt, gamma)* CE
loss = K.sum(FL, axis=1)
 return loss<compute_test_metric>
df['v2a1'].fillna(-1, inplace=True)
 df['v18q1'].fillna(0, inplace=True)
 df['SQBmeaned'].fillna(-1, inplace=True)
 df['rez_esc'].fillna(-1, inplace=True)
 df.meaneduc.fillna(df.meaneduc.median() , inplace=True)
beta_f2=2
def f2(y_true, y_pred):
tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=1)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=1)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=1)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=1)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + K.epsilon())
 f2 = tf.where(tf.is_nan(f2), tf.zeros_like(f2), f2)
 return K.mean(f2 )<train_model>
df.dependency.value_counts().head(5 )
class data_generator(Sequence):
def create_train(dataset_info, batch_size, shape, augument=True):
assert shape[2] == 3
while True:
dataset_info = shuffle(dataset_info)
 for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(preprocess_input(image))
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def create_valid(dataset_info, batch_size, shape, augument=False):
assert shape[2] == 3
while True:
for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(preprocess_input(image))
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def load_image(path, shape):
image = cv2.imread(path+'.png')
 image = cv2.resize(image,(SIZE, SIZE))
return image
def augment(image):
augment_img = iaa.Sequential([
iaa.SomeOf(( 0,4),[
iaa.Crop(percent=(0, 0.25)) ,
iaa.ContrastNormalization(( 0.8, 1.2)) ,
iaa.Multiply(( 0.9, 1.1), per_channel=0.2),
iaa.Fliplr(0.5),
])], random_order=True)
 image_aug = augment_img.augment_image(image)
 return image_aug<import_modules>
df.edjefe.value_counts().head(5 )
BatchNormalization, Input, Conv2D, Concatenate)
<choose_model_class>
df.edjefa.value_counts().head(5 )
def ResNet(stack_fn,
preact,
use_bias,
model_name='resnet',
include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):

 global backend, layers, models, keras_utils
backend, layers, models, keras_utils = keras.backend, keras.layers, keras.models, keras.utils
if not(weights in {'imagenet', None} or os.path.exists(weights)) :
raise ValueError('The `weights` argument should be either '
'`None`(random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
 if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
 input_shape = utils._obtain_input_shape(input_shape,
default_size=224,
min_size=32,
data_format=backend.image_data_format() ,
require_flatten=include_top,
weights=weights)
 if input_tensor is None:
img_input = layers.Input(shape=input_shape)
 else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
 else:
img_input = input_tensor
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
x = layers.ZeroPadding2D(padding=(( 3, 3),(3, 3)) , name='conv1_pad' )(img_input)
 x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv' )(x)
 if preact is False:
x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='conv1_bn' )(x)
 x = layers.Activation('relu', name='conv1_relu' )(x)
 x = layers.ZeroPadding2D(padding=(( 1, 1),(1, 1)) , name='pool1_pad' )(x)
 x = layers.MaxPooling2D(3, strides=2, name='pool1_pool' )(x)
 x = stack_fn(x)
 if preact is True:
x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='post_bn' )(x)
 x = layers.Activation('relu', name='post_relu' )(x)
 if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 x = layers.Dense(classes, activation='softmax', name='probs' )(x)
 else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 elif pooling == 'max':
x = layers.GlobalMaxPooling2D(name='max_pool' )(x)
 if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
 else:
inputs = img_input
model = models.Model(inputs, x, name=model_name)
 if(weights == 'imagenet')and(model_name in WEIGHTS_HASHES):
if include_top:
file_name = model_name + '_weights_tf_dim_ordering_tf_kernels.h5'
file_hash = WEIGHTS_HASHES[model_name][0]
else:
file_name = model_name + '_weights_tf_dim_ordering_tf_kernels_notop.h5'
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = keras_utils.get_file(file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
 model.load_weights(weights_path)
 elif weights is not None:
model.load_weights(weights)
 return model<categorify>
valores_replace = {'yes': 1, 'no': 0}
df['dependency'] = df['dependency'].replace(valores_replace ).astype(float)
 df['edjefe'] = df['edjefe'].replace(valores_replace ).astype(int)
 df['edjefa'] = df['edjefa'].replace(valores_replace ).astype(int )
def block2(x, filters, kernel_size=3, stride=1,
conv_shortcut=False, name=None):

 bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
preact = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name=name + '_preact_bn' )(x)
 preact = layers.Activation('relu', name=name + '_preact_relu' )(preact)
 if conv_shortcut is True:
shortcut = layers.Conv2D(4 * filters, 1, strides=stride,
name=name + '_0_conv' )(preact)
 else:
shortcut = layers.MaxPooling2D(1, strides=stride )(x)if stride > 1 else x
x = layers.Conv2D(filters, 1, strides=1, use_bias=False,
name=name + '_1_conv' )(preact)
 x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name=name + '_1_bn' )(x)
 x = layers.Activation('relu', name=name + '_1_relu' )(x)
 x = layers.ZeroPadding2D(padding=(( 1, 1),(1, 1)) , name=name + '_2_pad' )(x)
 x = layers.Conv2D(filters, kernel_size, strides=stride,
use_bias=False, name=name + '_2_conv' )(x)
 x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name=name + '_2_bn' )(x)
 x = layers.Activation('relu', name=name + '_2_relu' )(x)
 x = layers.Conv2D(4 * filters, 1, name=name + '_3_conv' )(x)
 x = layers.Add(name=name + '_out' )([shortcut, x])
 return x
def stack2(x, filters, blocks, stride1=2, name=None):

 x = block2(x, filters, conv_shortcut=True, name=name + '_block1')
 for i in range(2, blocks):
x = block2(x, filters, name=name + '_block' + str(i))
x = block2(x, filters, stride=stride1, name=name + '_block' + str(blocks))
return x
def ResNet50V2(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
def stack_fn(x):
x = stack2(x, 64, 3, name='conv2')
 x = stack2(x, 128, 4, name='conv3')
 x = stack2(x, 256, 6, name='conv4')
 x = stack2(x, 512, 3, stride1=1, name='conv5')
 return x
return ResNet(stack_fn, True, True, 'resnet50v2',
include_top, weights,
input_tensor, input_shape,
pooling, classes,
**kwargs )<load_pretrained>
df[df['Target'].notnull() ].corr() ['Target'].sort_values(ascending=False )
def create_model_resnet50(n_out=NUM_CLASSES):
print(RESNET_VER)
 if RESNET_VER == 1:
base_model =ResNet50(weights=None, include_top=False)
 base_model.load_weights('.. /input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
 else:
base_model =ResNet50V2(weights=None, include_top=False)
 base_model.load_weights('.. /input/keras-pretrain-model-weights/resnet50v2_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dense(1024, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(512, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation='sigmoid' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model<choose_model_class>
df["m2perpessoa"]=df["tamhog"]/df["tamviv"]
df['rentperpessoa'] = df['v2a1'] / df['tamviv']
model = create_model_resnet50(
 n_out=NUM_CLASSES)
 for layer in model.layers:
layer.trainable = False
for i in range(-10,0):
model.layers[i].trainable = True
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(LR_WARM),metrics=['acc',f2])
 model.summary() 
<choose_model_class>
print(round(df[df['cielorazo']==0]['idhogar'].nunique() /df[df['cielorazo']==1]['idhogar'].nunique() *100,2),'%' )
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 checkpoint = ModelCheckpoint(checkpoint_file, monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = False)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=LR_FACTOR, patience=PATIENCE,
verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=1e-5)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 indexes = np.arange(train_dataset_info.shape[0])
 train_indexes, valid_indexes = train_test_split(indexes, test_size=SPLIT_RATIO, random_state=8)
 train_generator = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(SIZE,SIZE,3), augument=True)
 train_generator_warmup = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(SIZE,SIZE,3), augument=False)
 validation_generator = data_generator.create_valid(
 train_dataset_info[valid_indexes], batch_size,(SIZE,SIZE,3), augument=False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]<define_variables>
print('** Métodos de se jogar lixo fora **')
 print('Tanker truck:',df[df['elimbasu1']==1].groupby(['elimbasu1'])['idhogar'].nunique().iloc[0])
 print('Botan hollow or buried:',df[df['elimbasu2']==1].groupby(['elimbasu2'])['idhogar'].nunique().iloc[0])
 print('Burning:',df[df['elimbasu3']==1].groupby(['elimbasu3'])['idhogar'].nunique().iloc[0])
 print('Throwing in an unoccupied space:',df[df['elimbasu4']==1].groupby(['elimbasu4'])['idhogar'].nunique().iloc[0])
 print('Throwing in river, creek or sea:',df[df['elimbasu5']==1].groupby(['elimbasu5'])['idhogar'].nunique().iloc[0])
 print('Other:',df[df['elimbasu6']==1].groupby(['elimbasu6'])['idhogar'].nunique().iloc[0])
print(np.ceil(float(len(train_indexes)) / float(batch_size)))
 print(np.ceil(float(len(valid_indexes)) / float(batch_size)))
 print(len(train_indexes), batch_size )<train_model>
feats = [c for c in df.columns if c not in ['Id', 'idhogar', 'Target']]
train, test = df[~df['Target'].isnull() ], df[df['Target'].isnull() ]
train.shape, test.shape
model.fit_generator(
 train_generator_warmup,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
epochs=1,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
verbose=1 )<find_best_params>
param_grid = {
'criterion' : ['gini', 'entropy'],
'class_weight' : ['balanced','balanced_subsample'],
'n_estimators': [100, 200, 300, 400]
}
rf = RandomForestClassifier()
grid_search = GridSearchCV(estimator = rf, param_grid = param_grid,
cv = 3, n_jobs = -1, verbose = 2 )
print(K.eval(model.optimizer.lr))<train_model>
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200 ,random_state=42)
 rf.fit(train[feats], train['Target'] )
for layer in model.layers:
layer.trainable = True
model.compile(
 loss=focal_loss,
optimizer=Adam(lr=LR),
metrics=['acc',f2])
 hist = model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
validation_data=validation_generator,
validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)) ,
epochs=epochs,
verbose=1,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
callbacks=callbacks_list )<find_best_params>
test['Target'] = rf.predict(test[feats] ).astype(int)
 print(rf.feature_importances_)
 print(train.columns )
print(K.eval(model.optimizer.lr))<load_pretrained>
test['Target'].value_counts(normalize=True )
print(os.listdir('.. /working/'))
model.load_weights(checkpoint_file )<categorify>
test[['Id', 'Target']].to_csv('submission1.csv', index=False )
BATCH = 512
fullValGen = data_generator.create_valid(
 train_dataset_info[valid_indexes], BATCH,(SIZE,SIZE,3))
n_val = round(train_dataset_info.shape[0]*0.2)//BATCH
print(n_val)
 lastFullValPred = np.empty(( 0, NUM_CLASSES))
lastFullValLabels = np.empty(( 0, NUM_CLASSES))
for i in tqdm(range(n_val+1)) :
im, lbl = next(fullValGen)
 scores = model.predict(im)
 lastFullValPred = np.append(lastFullValPred, scores, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape )<compute_test_metric>
heads2 = train[train['parentesco1'] == 1]
rf4 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 rf4.fit(heads2[feats], heads2['Target'])
 test['Target'] = rf4.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
def my_f2(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1),axis=1)
 tn = np.sum(( y_true == 0)&(y_pred == 0),axis=1)
 fp = np.sum(( y_true == 0)&(y_pred == 1),axis=1)
 fn = np.sum(( y_true == 1)&(y_pred == 0),axis=1)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + 1e-15)
 return np.mean(f2)
 def find_best_fixed_threshold(preds, targs, do_plot=True):
score = []
thrs = np.arange(0, 0.5, 0.01)
 for thr in tqdm(thrs):
score.append(my_f2(targs,(preds > thr ).astype(int)))
 score = np.array(score)
 pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')
 if do_plot:
plt.plot(thrs, score)
 plt.vlines(x=best_thr, ymin=score.min() , ymax=score.max())
 plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);
plt.show()
return best_thr, best_score<compute_train_metric>
xgb = XGBClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 xgb.fit(heads2[feats], heads2['Target'])
 test['Target'] = xgb.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission2.csv', index=False)
best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True )<load_from_csv>
abc = AdaBoostClassifier(random_state=42, n_estimators=700,learning_rate=1.0)
 abc.fit(heads2[feats], heads2['Target'])
 test['Target'] = abc.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission3.csv', index=False )
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 predicted = []
for i, name in tqdm(enumerate(submit['id'])) :
path = os.path.join('.. /input/imet-2019-fgvc6/test/', name)
 image = data_generator.load_image(path,(SIZE,SIZE,3))
score_predict = model.predict(preprocess_input(image[np.newaxis]))
label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted.append(str_predict_label )<save_to_csv>
rfc = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 scores = cross_val_score(rfc, heads2[feats], heads2['Target'], cv=5, n_jobs=-1)
 scores
submit['attribute_ids'] = predicted
if POST_PROCESS:
submit.to_csv('submission_not_process.csv', index=False)
 else:
submit.to_csv('submission.csv', index=False )<define_variables>
train['Target'].value_counts()
id_rare = set([199,81,221,230,366,11,104,396,262,805,112,146,281,
293,328,987,71,271,201,240,108,142,389,904,312,187])
 id_bool =(lastFullValPred[0] > 2)
 print(sum(id_bool))
for ii in id_rare:
id_bool[ii] = True
print(sum(id_bool))
lastFullValPred2 = lastFullValPred.copy()
lastFullValPred2[0][id_bool] = 0
thresh = np.percentile(lastFullValPred2[0],99.2)
 print(sum(lastFullValPred2[0] < thresh))
lastFullValPred2[0][lastFullValPred2[0] < thresh] = 0
print(sum(lastFullValPred2[0] == 0))<feature_engineering>
df_1 = train[train['Target'] == 1]
df_2 = train[train['Target'] == 2]
df_3 = train[train['Target'] == 3]
df_4 = train[train['Target'] == 4]
df_1.shape,df_2.shape,df_3.shape,df_4.shape,
print(lastFullValPred2[0].shape,(lastFullValPred2[i] < 0.1 ).shape)
 lastFullValPred2 = lastFullValPred.copy()
for i in tqdm(range(len(lastFullValPred2))):
lastFullValPred2[i][id_bool] = 0
thresh = np.percentile(lastFullValPred2[i],99.2)
 lastFullValPred2[i][lastFullValPred2[i] < thresh] = 0<compute_train_metric>
df_1 = resample(df_1,
replace=True,
n_samples=len(df_4),
random_state=42)
 df_2 = resample(df_2,
replace=True,
n_samples=len(df_4),
random_state=42)
 df_3 = resample(df_3,
replace=True,
n_samples=len(df_4),
random_state=42 )
best_thr, best_score = find_best_fixed_threshold(lastFullValPred2, lastFullValLabels, do_plot=True )<load_from_csv>
train = pd.concat([df_1, df_2, df_3, df_4] )
%%time
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 predicted_post = []
for i, name in tqdm(enumerate(submit['id'])) :
path = os.path.join('.. /input/imet-2019-fgvc6/test/', name)
 image = data_generator.load_image(path,(SIZE,SIZE,3))
score_predict = model.predict(preprocess_input(image[np.newaxis]))
thresh = np.percentile(score_predict,99.2)
 score_predict[score_predict < thresh] = 0
label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted_post.append(str_predict_label )<save_to_csv>
heads2 = train[train['parentesco1'] == 1]
rf4 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 rf4.fit(heads2[feats], heads2['Target'])
 test['Target'] = rf4.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission4.csv', index=False )
submit['attribute_ids'] = predicted_post
if POST_PROCESS:
submit.to_csv('submission.csv', index=False)
 else:
submit.to_csv('submission_post_process.csv', index=False)
 submit2 = submit.copy()
submit2.n_cate = submit.attribute_ids.apply(lambda x: len(x.split(" ")))
 _ = submit2.n_cate.value_counts().sort_index().plot.bar()<import_modules>
rf4 = RandomForestClassifier(n_jobs=-1, n_estimators=200 ,random_state=42)
 rf4.fit(heads2[feats], heads2['Target'])
 test['Target'] = rf4.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission5.csv', index=False )