Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<load_pretrained>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from skimage.io import imshow, imsave
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import TruncatedSVD
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
import lightgbm as lgb
import matplotlib.pyplot as plt
import seaborn as sns
import os, sys, time
from bayes_opt import BayesianOptimization
ckpt_path = "/kaggle/input/pytorch-multi-label-classification-effnet-b7/efficientnet_b7_model.pth"
device = torch.device("cuda")
 clf_efficientnet_b7 = EfficientNet.from_pretrained('efficientnet-b7', num_classes=4 ).cuda()
clf_efficientnet_b7.to(device)
 clf_efficientnet_b7.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 clf_efficientnet_b7.load_state_dict(state["state_dict"] )<choose_model_class>
Costatrain = pd.read_csv(".. /input/train.csv")
 Costatest = pd.read_csv(".. /input/test.csv")
 print("Train Dataset: Rows, Columns: ", Costatrain.shape)
 print("Test Dataset: Rows, Columns: ", Costatest.shape )
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([
unet_se_resnext50_32x4d,
unet_mobilenet2,
unet_resnet34,
unet_efficientnet,
senext50_30_epochs_high_threshold
])
 model_clf = Model([
clf_efficientnet_b0,
clf_efficientnet_b4,
clf_efficientnet_b5,
clf_efficientnet_b7,
clf_resnet34
] )<categorify>
Train_target = Costatrain['Target']
Train_target.value_counts(normalize=True )
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<define_search_model>
naData = Costatrain.isnull().sum().values / Costatrain.shape[0] *100
df_na = pd.DataFrame(naData, index=Costatrain.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_count} rows which have missing value in train set ')
 df_na.head(10)
def close(mask):

 kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(5,5))
closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
 return closing<define_search_space>
naData = Costatest.isnull().sum().values / Costatest.shape[0] *100
df_na = pd.DataFrame(naData, index=Costatest.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_count} rows which have missing value in test set ')
 df_na.head(10)
thresholds = [0.5, 0.75, 0.5, 0.75]
clf_thresholds = [0.5, 0.5, 0.75, 0.85]
min_area = [600, 1000, 1200, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
features_no_tta = None
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
if i == 0:
features_no_tta = features
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
clf_preds = torch.sigmoid(model_clf(features_no_tta)).detach().cpu().numpy()
for p, clf_pred, file in zip(preds, clf_preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 if clf_pred[i] <= clf_thresholds[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 else:
p_channel = close(p_channel)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 } )<save_to_csv>
Costatrain['v2a1'] = Costatrain['v2a1'].fillna(value=Costatrain['tipovivi3'])
 Costatest['v2a1'] = Costatest['v2a1'].fillna(value=Costatest['tipovivi3'])
 Costatrain['v18q1'] = Costatrain['v18q1'].fillna(value=Costatrain['v18q'])
 Costatest['v18q1'] = Costatest['v18q1'].fillna(value=Costatest['v18q'])
 Costatrain.info()
Costatest.info()
Costatrain.head(5)
 Costatest.head(5 )
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )<feature_engineering>
cols = ['edjefe', 'edjefa']
Costatrain[cols] = Costatrain[cols].replace({'no': 0, 'yes':1} ).astype(float)
 Costatest[cols] = Costatest[cols].replace({'no': 0, 'yes':1} ).astype(float)
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()<feature_engineering>
cols=['techozinc','techoentrepiso','techocane','techootro']
Costatrain['roof_waste_material'] = np.nan
Costatest['roof_waste_material'] = np.nan
Costatrain['electricity_other'] = np.nan
Costatest['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
Costatrain['roof_waste_material'] = Costatrain.apply(lambda x : fill_roof_exception(x),axis=1)
 Costatest['roof_waste_material'] = Costatest.apply(lambda x : fill_roof_exception(x),axis=1)
 Costatrain['electricity_other'] = Costatrain.apply(lambda x : fill_no_electricity(x),axis=1)
 Costatest['electricity_other'] = Costatest.apply(lambda x : fill_no_electricity(x),axis=1)
%matplotlib inline
df = pd.read_csv('submission.csv')[:40]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()<categorify>
Costatrain['adult'] = Costatrain['hogar_adul'] - Costatrain['hogar_mayor']
Costatrain['dependency_count'] = Costatrain['hogar_nin'] + Costatrain['hogar_mayor']
Costatrain['dependency'] = Costatrain['dependency_count'] / Costatrain['adult']
Costatrain['child_percent'] = Costatrain['hogar_nin']/Costatrain['hogar_total']
Costatrain['elder_percent'] = Costatrain['hogar_mayor']/Costatrain['hogar_total']
Costatrain['adult_percent'] = Costatrain['hogar_adul']/Costatrain['hogar_total']
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
return predictions<load_from_zip>
Costatest['adult'] = Costatest['hogar_adul'] - Costatest['hogar_mayor']
Costatest['dependency_count'] = Costatest['hogar_nin'] + Costatest['hogar_mayor']
Costatest['dependency'] = Costatest['dependency_count'] / Costatest['adult']
Costatest['child_percent'] = Costatest['hogar_nin']/Costatest['hogar_total']
Costatest['elder_percent'] = Costatest['hogar_mayor']/Costatest['hogar_total']
Costatest['adult_percent'] = Costatest['hogar_adul']/Costatest['hogar_total']
!mkdir -p /tmp/pip/cache/
!cp /kaggle/input/segmentation-models-zip-003/efficientnet_pytorch-0.4.0.xyz /tmp/pip/cache/efficientnet_pytorch-0.4.0.tar.gz
!cp /kaggle/input/segmentation-models-zip-003/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp /kaggle/input/segmentation-models-zip-003/segmentation_models_pytorch-0.0.3.xyz /tmp/pip/cache/segmentation_models_pytorch-0.0.3.tar.gz<install_modules>
submission = Costatest[['Id']]
Costatrain.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 Costatest.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 correlation = Costatrain.corr()
correlation = correlation['Target'].sort_values(ascending=False)
 print(f'The most 20 positive feature:
{correlation.head(20)}')
 print('*'*50)
 print(f'The most 20 negative feature:
{correlation.tail(20)}' )
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch
!pip install /kaggle/input/tta-pytorch/ttach-0.0.1-py3-none-any.whl<set_options>
y = Costatrain['Target']
Costatrain.drop(['Target'], inplace=True, axis=1)
warnings.filterwarnings('ignore')
<load_pretrained>
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.89, min_child_samples = 90, num_leaves = 14, subsample = 0.96)
 kfold = 5
skf = StratifiedKFold(n_splits=kfold, shuffle=True)
 X=Costatrain
folds = skf.split(X,y)
 type(folds)
 predicts = []
for train_index, test_index in folds:
print("train_test_index")
 X_train, X_val = Costatrain.iloc[train_index], Costatrain.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=400, verbose=100)
 predicts.append(clf.predict(Costatest))
<normalization><EOS> 
y_pred =np.array(predicts ).mean(axis=0 ).round().astype(int)
 sub = pd.DataFrame()
sub['Id'] = submission['Id']
sub['Target'] = y_pred
sub.to_csv('submission.csv', index=False)
 sub.head()