Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<feature_engineering>
import numpy as np
import pandas as pd
names_arr = label_df.attribute_name.values
df_train["attribute_names"] = [", ".join(names_arr[arr == 1])for arr in train_attr_ohot]<feature_engineering>
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape
df_train["attr_num"] = train_attr_ohot.sum(axis=1)
 df_train["culture_attr_num"] = train_attr_ohot[:, :398].sum(axis=1)
 df_train["tag_attr_num"] = train_attr_ohot[:, 398:].sum(axis=1 )<feature_engineering>
df_all = df.append(test )
i = 1
df_train["attribute_ids"]=df_train["attribute_ids"].apply(lambda x:x.split(" "))
df_train["id"]=df_train["id"].apply(lambda x:x+".png")
 plt.figure(figsize=[30,30])
 for img_name in os.listdir(".. /input/imet-2019-fgvc6/train/")[5:10]:
img = cv2.imread(".. /input/imet-2019-fgvc6/train/%s" % img_name)[...,[2, 1, 0]]
plt.subplot(5, 1, i)
 plt.imshow(img)
 ids = df_train[df_train["id"] == img_name]["attribute_ids"]
print(ids)
 title_val = []
for tag_id in ids.values[0]:
att_name = label_df[label_df['attribute_id'].astype(str)== tag_id]['attribute_name'].values[0]
title_val.append(att_name)
 plt.title(title_val)
 i += 1
plt.show()<import_modules>
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int)
 df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float)
 df_all['v2a1'].fillna(-1, inplace=True)
 df_all['v18q1'].fillna(0, inplace=True)
 df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True)
 feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
train, test = df_all[~df_all['Target'].isnull() ], df_all[df_all['Target'].isnull() ]
import os
import numpy as np
from PIL import Image
import pandas as pd
import hashlib<compute_test_metric>
forest = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced')
 forest.fit(train[feats], train['Target'])
 test['Target'] = forest.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
def check_md5(fname):
hash_md5 = hashlib.md5()
with open(fname, "rb")as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
 return hash_md5.hexdigest()<define_variables>
feats2 = [c for c in list(dict(teste.head(10)).keys())if c not in ['Id', 'idhogar', 'Target']]
forest2 = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced' )
file_names = []
path_root = '.. /input/imet-2019-fgvc6/train/'
for filename in os.listdir(path_root)[0:20]:
file_names.append(check_md5(path_root+filename))
print(len(file_names))<count_unique_values>
forest.fit(train[feats2], train['Target'])
 test['Target'] = forest.predict(test[feats2] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
unit = np.unique(file_names,return_counts=True)
 count = 0
for i in range(len(unit[1])) :
if unit[1][i]>1:
count += 1
print('Duplicated Images')
 if count == 0:
print('NOT Duplicated Images' )<train_model>
retirar.append('Id')
 retirar.append('idhogar')
 retirar.append('Target' )
def padding_image(path_img):
image_old = Image.open(path_img)
 width, height = image_old.size
if width > height:
distance_max = width
array = np.zeros([distance_max, distance_max, 3], dtype=np.uint8)
 array.fill(0)
 image_new = Image.fromarray(array)
 xmin = 0
ymin = int(( distance_max / 2)-(height / 2))
xmax = distance_max
ymax = int(( distance_max / 2)+(height / 2))
image_new.paste(image_old,(xmin, ymin, xmax, ymax))
return image_new
elif width < height:
distance_max = height
array = np.zeros([distance_max, distance_max, 3], dtype=np.uint8)
 array.fill(0)
 image_new = Image.fromarray(array)
 xmin = int(( distance_max / 2)-(width / 2))
ymin = 0
xmax = int(( distance_max / 2)+(width / 2))
ymax = distance_max
image_new.paste(image_old,(xmin, ymin, xmax, ymax))
return image_new
else:
return image_old<import_modules>
feats3 = [c for c in df_all.columns if c not in retirar]
train, test = df_all[~df_all['Target'].isnull() ], df_all[df_all['Target'].isnull() ]
fastai.__version__<define_variables>
forest = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced')
 forest.fit(train[feats3], train['Target'])
 test['Target'] = forest.predict(test[feats3] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
BATCH = 126
SIZE = 250
path = Path('.. /input/imet-2019-fgvc6/' )<load_pretrained>
retirar = list()
colunas_escolhidas = list()
while len(retirar)< 10:
index_coluna = random.randint(0,140)
 coluna = teste.index[index_coluna]
if coluna not in retirar:
retirar.append(coluna)
 else:
continue
retirar.append('Id')
 retirar.append('idhogar')
 retirar.append('Target' )
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/resnet50/resnet50.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'resnet50.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url<load_from_csv>
feats4 = [c for c in df_all.columns if c not in retirar]
train, test = df_all[~df_all['Target'].isnull() ], df_all[df_all['Target'].isnull() ]
train_df = pd.read_csv(path/'train.csv')
 train_df.head()<load_from_csv>
forest = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced')
 forest.fit(train[feats4], train['Target'])
 test['Target'] = forest.predict(test[feats4] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )