Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<normalization>
import numpy as np
import pandas as pd
from umap import UMAP
from sklearn.decomposition import PCA, FastICA
from sklearn.manifold import TSNE
import gc
import featuretools as ft
import featuretools.variable_types as vtypes
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import Imputer
import lightgbm as lgb
from sklearn.model_selection import KFold, train_test_split, StratifiedKFold
from sklearn.metrics import roc_auc_score, f1_score
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )<categorify>
feature_matrix = pd.read_csv('.. /input/costa-rican-poverty-derived-data/ft_2000.csv')
 feature_matrix.shape
def create_transforms(additional):
res = list(additional)
 res.extend([
albu.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = albu.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[albu.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<define_search_space>
feature_matrix['SUM(ind.rez_esc / escolari)'] = feature_matrix['SUM(ind.rez_esc / escolari)'].astype(np.float32)
 feature_matrix['SUM(ind.age / escolari)'] = feature_matrix['SUM(ind.age / escolari)'].astype(np.float32 )
thresholds = [0.50, 0.50, 0.50, 0.50]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel = post_process(p_channel, thresholds[i], min_area[i])
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df.to_csv('submission.csv', index=False)	<save_to_csv>
for col in feature_matrix:
if feature_matrix[col].dtype == 'object':
if col != 'idhogar':
feature_matrix[col] = feature_matrix[col].astype(np.float32 )
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )<feature_engineering>
feature_matrix.columns[np.where(feature_matrix.dtypes == 'object')]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()<set_options>
missing_threshold = 0.95
correlation_threshold = 0.99
train = feature_matrix[feature_matrix['Target'].notnull() ]
test = feature_matrix[feature_matrix['Target'].isnull() ]
train_ids = list(train.pop('idhogar'))
test_ids = list(test.pop('idhogar'))
feature_matrix = feature_matrix.replace({np.inf: np.nan, -np.inf:np.nan})
 n_features_start = feature_matrix.shape[1]
print('Original shape: ', feature_matrix.shape)
 missing = pd.DataFrame(feature_matrix.isnull().sum())
 missing['fraction'] = missing[0] / feature_matrix.shape[0]
missing.sort_values('fraction', ascending = False, inplace = True)
 missing_cols = list(missing[missing['fraction'] > missing_threshold].index)
 n_missing_cols = len(missing_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in missing_cols]]
print('{} missing columns with threshold: {}.'.format(n_missing_cols, missing_threshold))
unique_counts = pd.DataFrame(feature_matrix.nunique() ).sort_values(0, ascending = True)
 zero_variance_cols = list(unique_counts[unique_counts[0] == 1].index)
 n_zero_variance_cols = len(zero_variance_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in zero_variance_cols]]
print('{} zero variance columns.'.format(n_zero_variance_cols))
corr_matrix = feature_matrix.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
n_collinear = len(to_drop)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in to_drop]]
print('{} collinear columns removed with correlation above {}.'.format(n_collinear, correlation_threshold))
total_removed = n_missing_cols + n_zero_variance_cols + n_collinear
print('Total columns removed: ', total_removed)
 print('Shape after feature selection: {}.'.format(feature_matrix.shape))
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
train = feature_matrix[feature_matrix['Target'].notnull() ]
test = feature_matrix[feature_matrix['Target'].isnull() ]
train_ids = list(train.pop('idhogar'))
test_ids = list(test.pop('idhogar'))
train_labels = np.array(train.pop('Target')).reshape(( -1,))
test = test.drop(columns = 'Target')
 train = train.replace({np.inf: np.nan, -np.inf: np.nan})
 test = test.replace({np.inf: np.nan, -np.inf: np.nan} )
warnings.filterwarnings('ignore')
<load_pretrained>
feature_list = list(train.columns)
 imputer = SimpleImputer(strategy = 'median')
 train = imputer.fit_transform(train)
 test = imputer.transform(test)
 train_df = pd.DataFrame(train, columns = feature_list)
 test_df = pd.DataFrame(test, columns = feature_list)
 train.shape
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()<normalization>
train_df = train_df.astype(np.float32)
 test_df = test_df.astype(np.float32 )
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )<categorify>
n_components = 3
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<define_search_space>
umap = UMAP(n_components=n_components)
 pca = PCA(n_components=n_components)
 ica = FastICA(n_components=n_components)
 tsne = TSNE(n_components=n_components )
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df.to_csv('submission.csv', index=False)	<save_to_csv>
for method, name in zip([umap, pca, ica], ['umap', 'pca', 'ica']):
if name == 'umap':
start = timer()
reduction = method.fit_transform(train, train_labels)
 test_reduction = method.transform(test)
 end = timer()
else:
start = timer()
reduction = method.fit_transform(train)
 test_reduction = method.transform(test)
 end = timer()
print(f'Method: {name} {round(end - start, 2)} seconds elapsed.')
 train_df['%s_c1' % name] = reduction[:, 0]
train_df['%s_c2' % name] = reduction[:, 1]
train_df['%s_c3' % name] = reduction[:, 2]
test_df['%s_c1' % name] = test_reduction[:, 0]
test_df['%s_c2' % name] = test_reduction[:, 1]
test_df['%s_c3' % name] = test_reduction[:, 2]
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )<feature_engineering>
train_df['label'] = train_labels
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()<feature_engineering>
test_comp = pd.read_csv('.. /input/costa-rican-household-poverty-prediction/test.csv')
 submission_base = test_comp.loc[:, ['idhogar', 'Id']]
%matplotlib inline
df = pd.read_csv('submission.csv')[:40]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()<install_modules>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscripts/'
sys.path.append(package_path )<import_modules>
def model_gbm(features, labels, test_features, test_ids, nfolds = 5, return_preds = False):

 feature_names = list(features.columns)
 model = lgb.LGBMClassifier(boosting_type = 'gbdt', n_estimators = 10000, max_depth = -1,
learning_rate = 0.025, metric = 'None', min_child_samples = 30,
reg_alpha = 0.35, reg_lambda = 0.6, num_leaves = 15,
colsample_bytree = 0.85, objective = 'multiclass',
class_weight = 'balanced',
n_jobs = -1)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 print(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']],
on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(VerticalFlip, HorizontalFlip, ShiftScaleRotate, Normalize, Resize, Compose, GaussNoise)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
from resnet_unet import Unet<categorify>
for col in train_df:
if 'Target' in col:
print(col )
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<feature_engineering>
predictions, fi = model_gbm(train_df.drop(columns = 'label'), train_labels,
test_df, test_ids, return_preds = True )
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std,TTA=None):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 if TTA=="h":
self.transform = Compose(
 [
HorizontalFlip(p=1),
Normalize(mean=mean,std=std,p=1),
ToTensor() ,
] 
)
 elif TTA=="v":
self.transform = Compose(
 [
VerticalFlip(p=1),
Normalize(mean=mean,std=std,p=1),
ToTensor() ,
] 
)
 else:
self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
submission, fi, scores = model_gbm(train_df.drop(columns = 'label'), train_labels,
test_df, test_ids, return_preds = False)
 submission.to_csv('dimension_reduction.csv', index = False )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
for method, name in zip([umap, pca, ica], ['umap', 'pca', 'ica']):
start = timer()
reduction = method.fit_transform(train)
 test_reduction = method.transform(test)
 end = timer()
print(f'Method: {name} {round(end - start, 2)} seconds elapsed.')
 train_df['%s_c1' % name] = reduction[:, 0]
train_df['%s_c2' % name] = reduction[:, 1]
train_df['%s_c3' % name] = reduction[:, 2]
test_df['%s_c1' % name] = test_reduction[:, 0]
test_df['%s_c2' % name] = test_reduction[:, 1]
test_df['%s_c3' % name] = test_reduction[:, 2]
<load_from_csv><EOS> 
submission, fi, scores = model_gbm(train_df.drop(columns = 'label'), train_labels,
test_df, test_ids, return_preds = False)
 submission.to_csv('dimension_reduction_nolabels.csv', index = False )