Kaggle competition: Costa Rican Household Poverty Level Prediction
import os
import json
import cv2
import keras
from keras import backend as K
from keras.models import Model
from keras.layers import Input
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.losses import binary_crossentropy
from keras.callbacks import Callback, ModelCheckpoint
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split<load_from_csv>
data=pd.read_csv('.. /input/train.csv' )
train_df_origin = pd.read_csv('.. /input/severstal-steel-defect-detection/train.csv')
 train_df_origin.shape<create_dataframe>
data[(data['dependency']=='no')&(data['SQBdependency']!=0)]
train_df = pd.DataFrame(columns=['ImageId_ClassId','EncodedPixels'])
 for i in tqdm(range(len(train_fns_split))):
for j in range(4):
tmp_se = pd.Series([train_fns_split[4][i]+'_{}'.format(j+1),None], index=train_df.columns)
 train_df = train_df.append(tmp_se, ignore_index=True)
 train_df.head()<feature_engineering>
data[(data['dependency']=='yes')&(data['SQBdependency']!=1)]
for i in tqdm(range(len(train_df_origin))):
imageid_classid = train_df_origin['ImageId'][i]+'_{}'.format(train_df_origin['ClassId'][i])
 idx=train_df.query('ImageId_ClassId == "{}"'.format(imageid_classid)).index[0]
train_df['EncodedPixels'][idx] = train_df_origin['EncodedPixels'][i]
train_df.head()<feature_engineering>
data[(data['dependency']=='3')&(data['SQBdependency']!=9)]
train_df['ImageId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 train_df['ClassId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[1])
 train_df['hasMask'] = ~ train_df['EncodedPixels'].isna()
print(train_df.shape)
 train_df.head()<groupby>
data['dependency']=np.sqrt(data['SQBdependency'] )
mask_count_df = train_df.groupby('ImageId' ).agg(np.sum ).reset_index()
mask_count_df.sort_values('hasMask', ascending=False, inplace=True)
 print(mask_count_df.shape)
 mask_count_df.head()<load_from_csv>
data[['edjefe', 'edjefa', 'SQBedjefe']][data['edjefe']=='yes']
sub_df_origin = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df_origin.head()<create_dataframe>
data[(data['edjefe']=='yes')&(data['edjefa']!='no')]
sub_df = pd.DataFrame(columns=['ImageId_ClassId','EncodedPixels'])
 for i in tqdm(range(len(sub_df_origin))):
for j in range(4):
tmp_se = pd.Series([sub_df_origin['ImageId'][i]+'_{}'.format(j+1),None], index=sub_df.columns)
 sub_df = sub_df.append(tmp_se, ignore_index=True)
 sub_df.head(9 )<load_from_csv>
data[(data['edjefa']=='yes')&(data['parentesco1']==1)][['edjefe', 'edjefa', 'parentesco1', 'escolari']]
sub = pd.read_csv('.. /input/submain2/sub_main2.csv')
 sub_df['ImageId'] = sub_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['ImageId'].unique() , columns=['ImageId'] )<categorify>
data[data['edjefe']=='yes'][['edjefe', 'edjefa','age', 'escolari', 'parentesco1','male', 'female', 'idhogar']]
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(mask_rle, shape=(256,1600)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T<categorify>
data[(data['edjefe']=='no')&(data['edjefa']=='no')][['edjefe', 'edjefa', 'age', 'escolari', 'female', 'male', 'Id', 'parentesco1', 'idhogar']]
def build_masks(rles, input_shape):
depth = len(rles)
 height, width = input_shape
masks = np.zeros(( height, width, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle,(width, height))
return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles<compute_test_metric>
data[(data['edjefe']=='yes')& data['parentesco1']==1][['escolari']]
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )<train_model>
conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data[['edjefe', 'edjefa', 'edjefx']][:15]
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path='.. /input/severstal-steel-defect-detection/train_images',
batch_size=32, dim=(256, 1600), n_channels=1,
n_classes=4, random_state=2019, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
X = np.empty(( self.batch_size, *self.dim, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_grayscale(img_path)
 X[i,] = img
return X
def __generate_y(self, list_IDs_batch):
y = np.empty(( self.batch_size, *self.dim, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):
img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img<split>
data.columns[data.isna().sum() !=0]
BATCH_SIZE = 16
train_idx, val_idx = train_test_split(
 mask_count_df.index, random_state=2019, test_size=0.15 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4 
)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4
)<choose_model_class>
data[data['meaneduc'].isnull() ]
def build_model(input_shape):
inputs = Input(input_shape)
 c1 = Conv2D(8,(3, 3), activation='elu', padding='same' )(inputs)
 c1 = Conv2D(8,(3, 3), activation='elu', padding='same' )(c1)
 p1 = MaxPooling2D(( 2, 2))(c1)
 c2 = Conv2D(16,(3, 3), activation='elu', padding='same' )(p1)
 c2 = Conv2D(16,(3, 3), activation='elu', padding='same' )(c2)
 p2 = MaxPooling2D(( 2, 2))(c2)
 c3 = Conv2D(32,(3, 3), activation='elu', padding='same' )(p2)
 c3 = Conv2D(32,(3, 3), activation='elu', padding='same' )(c3)
 p3 = MaxPooling2D(( 2, 2))(c3)
 c4 = Conv2D(64,(3, 3), activation='elu', padding='same' )(p3)
 c4 = Conv2D(64,(3, 3), activation='elu', padding='same' )(c4)
 p4 = MaxPooling2D(pool_size=(2, 2))(c4)
 c5 = Conv2D(64,(3, 3), activation='elu', padding='same' )(p4)
 c5 = Conv2D(64,(3, 3), activation='elu', padding='same' )(c5)
 p5 = MaxPooling2D(pool_size=(2, 2))(c5)
 c55 = Conv2D(128,(3, 3), activation='elu', padding='same' )(p5)
 c55 = Conv2D(128,(3, 3), activation='elu', padding='same' )(c55)
 u6 = Conv2DTranspose(64,(2, 2), strides=(2, 2), padding='same' )(c55)
 u6 = concatenate([u6, c5])
 c6 = Conv2D(64,(3, 3), activation='elu', padding='same' )(u6)
 c6 = Conv2D(64,(3, 3), activation='elu', padding='same' )(c6)
 u71 = Conv2DTranspose(32,(2, 2), strides=(2, 2), padding='same' )(c6)
 u71 = concatenate([u71, c4])
 c71 = Conv2D(32,(3, 3), activation='elu', padding='same' )(u71)
 c61 = Conv2D(32,(3, 3), activation='elu', padding='same' )(c71)
 u7 = Conv2DTranspose(32,(2, 2), strides=(2, 2), padding='same' )(c61)
 u7 = concatenate([u7, c3])
 c7 = Conv2D(32,(3, 3), activation='elu', padding='same' )(u7)
 c7 = Conv2D(32,(3, 3), activation='elu', padding='same' )(c7)
 u8 = Conv2DTranspose(16,(2, 2), strides=(2, 2), padding='same' )(c7)
 u8 = concatenate([u8, c2])
 c8 = Conv2D(16,(3, 3), activation='elu', padding='same' )(u8)
 c8 = Conv2D(16,(3, 3), activation='elu', padding='same' )(c8)
 u9 = Conv2DTranspose(8,(2, 2), strides=(2, 2), padding='same' )(c8)
 u9 = concatenate([u9, c1], axis=3)
 c9 = Conv2D(8,(3, 3), activation='elu', padding='same' )(u9)
 c9 = Conv2D(8,(3, 3), activation='elu', padding='same' )(c9)
 outputs = Conv2D(4,(1, 1), activation='sigmoid' )(c9)
 model = Model(inputs=[inputs], outputs=[outputs])
 model.compile(optimizer='adam', loss=bce_dice_loss, metrics=[dice_coef])
 return model<define_variables>
data[data['meaneduc'].isnull() ][['Id','idhogar','edjefe','edjefa', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'age', 'escolari']]
model.load_weights('.. /input/modelunet3/model.h5')
 test_df = []
for i in range(0, test_imgs.shape[0], 500):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 500)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=sub_df,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model.predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 for j, b in tqdm(enumerate(batch_idx)) :
filename = test_imgs['ImageId'].iloc[b]
image_df = sub_df[sub_df['ImageId'] == filename].copy()
pred_masks = batch_pred_masks[j, ].round().astype(int)
 pred_rles = build_rles(pred_masks)
 image_df['EncodedPixels'] = pred_rles
test_df.append(image_df )<concatenate>
print(len(data[data['idhogar']==data.iloc[1291]['idhogar']]))
print(len(data[data['idhogar']==data.iloc[1840]['idhogar']]))
print(len(data[data['idhogar']==data.iloc[2049]['idhogar']]))
subb = pd.concat(test_df )<save_to_csv>
meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
sub.to_csv('submission.csv', index=False )<save_to_csv>
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
sub.to_csv('submission.csv', index=False )<set_options>
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
warnings.filterwarnings('ignore')
<load_pretrained>
data['v2a1'].isnull().sum()
unet = load('.. /input/severstaleffnet/traced_effnetb7_mixup_retrain.pth' ).cuda()
cls = load('.. /input/severstall-effnetb0-fimal-stage/traced_model.pth' ).cuda()
cls_alt = load('.. /input/severstaleffnet/traced_effnetb0_lovasz.pth' ).cuda()<categorify>
norent=data[data['v2a1'].isnull() ]
print("Owns his house:", norent[norent['tipovivi1']==1]['Id'].count())
 print("Owns his house paying installments", norent[norent['tipovivi2']==1]['Id'].count())
 print("Rented ", norent[norent['tipovivi3']==1]['Id'].count())
 print("Precarious ", norent[norent['tipovivi4']==1]['Id'].count())
 print("Other ", norent[norent['tipovivi5']==1]['Id'].count())
 print("Total ", 6860 )
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 1
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)],
[A.VerticalFlip(p=1)],
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<data_type_conversions>
data['v2a1']=data['v2a1'].fillna(0 )
l = torch.tensor([1, 0])
 if np.count_nonzero(l):
print('yay' )<save_to_csv>
data['v18q1'].isna().sum()
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )<feature_engineering>
tabletnan=data[data['v18q1'].isnull() ]
tabletnan[tabletnan['v18q']==0]['Id'].count()
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()<feature_engineering>
data['v18q1']=data['v18q1'].fillna(0 )
%matplotlib inline
df = pd.read_csv('submission.csv')[:40]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()<install_modules>
data['rez_esc'].isnull().sum()
!pip install.. /input/pretrainedmodels3/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript3'
sys.path.append(package_path )<import_modules>
data[data['rez_esc']>1][['age', 'escolari', 'rez_esc']][:20]
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F
from model import Unet<load_pretrained>
rez_esc_nan=data[data['rez_esc'].isnull() ]
rez_esc_nan[(rez_esc_nan['age']<18)& rez_esc_nan['escolari']>0][['age', 'escolari']]
model_classification = Resnet34_classification()
model_classification.load_state_dict(torch.load('.. /input/clsification3/00007500_model.pth', map_location=lambda storage, loc: storage), strict=True )<categorify>
data['rez_esc']=data['rez_esc'].fillna(0 )
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<define_variables>
d={}
weird=[]
for row in data.iterrows() :
idhogar=row[1]['idhogar']
target=row[1]['Target']
if idhogar in d:
if d[idhogar]!=target:
weird.append(idhogar)
 else:
d[idhogar]=target
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<define_search_space>
len(set(weird))
batch_size = 1
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225 )<load_from_csv>
data[data['idhogar']==weird[2]][['idhogar','parentesco1', 'Target']]
df = pd.read_csv(sample_submission_path )<create_dataframe>
for i in set(weird):
hhold=data[data['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
data.at[idx, 'Target']=target
testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=0,
pin_memory=True
)<define_variables>
data[data['idhogar']==weird[1]][['idhogar','parentesco1', 'Target']]
threshold_label = [0.50,0.50,0.50,0.50,]<define_variables>
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data
augment = ['null']<prepare_output>
%matplotlib inline
probability_label, image_id = get_classification_preds(model_classification, testset)
 predict_label = probability_label>np.array(threshold_label ).reshape(1,4,1,1)
 image_id_class_id = []
encoded_pixel = []
for b in range(len(image_id)) :
for c in range(4):
image_id_class_id.append(image_id[b]+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle ='1 1'
encoded_pixel.append(rle)
 df_classification = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'] )<load_pretrained>
data_undersampled=data.drop(data.query('Target == 4' ).sample(frac=.75 ).index )
ckpt_path = ".. /input/res18fullsize3/model_swa.pth"
device = torch.device("cuda")
 model_segmentation = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model_segmentation.to(device)
 model_segmentation.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model_segmentation.load_state_dict(state["state_dict"] )<define_variables>
X=data_undersampled.drop(['Id', 'idhogar', 'Target', 'edjefe', 'edjefa'], axis=1)
 y=data_undersampled['Target']
models = [model_segmentation]<categorify>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_search_space>
clf = RandomForestClassifier()
params={'n_estimators': list(range(40,61, 1)) }
gs = GridSearchCV(clf, params, cv=5 )
threshold_pixel = [0.5,0.5,0.5,0.5,]
min_size = [200,1500,1500,2000]<normalization>
gs.fit(X_train, y_train )
def null_augment(input): return input
def flip_lr_augment(input): return torch.flip(input, dims=[2])
 def flip_ud_augment(input): return torch.flip(input, dims=[3])
 def null_inverse_augment(logit): return logit
def flip_lr_inverse_augment(logit): return torch.flip(logit, dims=[2])
 def flip_ud_inverse_augment(logit): return torch.flip(logit, dims=[3])
 augment =(
(null_augment, null_inverse_augment), 
(flip_lr_augment,flip_lr_inverse_augment), 
(flip_ud_augment,flip_ud_inverse_augment),
)<find_best_params>
preds=gs.predict(X_test )
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
images = images.cuda()
batch_preds = 0
probabilities = []
for model in models:
model = model.cuda()
for k,(a, inv_a)in enumerate(augment):
logit = model(a(images))
p = inv_a(torch.sigmoid(logit))
if k ==0:
probability = p**0.5
else:
probability += p**0.5
probability = probability/len(augment)
 probabilities.append(probability)
 batch_preds+=probability
batch_preds = batch_preds.data.cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, threshold_pixel[cls], min_size[cls])
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df_segmentation = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'] )<create_dataframe>
print(classification_report(y_test, preds))
df= df_segmentation.copy()<data_type_conversions>
print(confusion_matrix(y_test, preds))
if 1:
df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<define_variables>
print(gs.best_params_)
 print(gs.best_score_)
 print(gs.best_estimator_ )
df_mask = df_segmentation.copy()
df_label = df_classification.copy()<feature_engineering>
cvres = gs.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(mean_score), params )
assert(np.all(df_mask['ImageId_ClassId'].values == df_label['ImageId_ClassId'].values))
print(( df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels'] != '' ).sum())
 df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels']=''<save_to_csv>
test_data=pd.read_csv('.. /input/test.csv' )
df_mask.to_csv("submission.csv", index=False )<data_type_conversions>
ids=test_data['Id']
test_data.drop(['Id', 'idhogar'], axis=1, inplace=True )
if 1:
df_mask['Class'] = df_mask['ImageId_ClassId'].str[-1].astype(np.int32)
 df_mask['Label'] =(df_mask['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df_mask['Class']==1)&(df_mask['Label']==1)).sum()
pos2 =(( df_mask['Class']==2)&(df_mask['Label']==1)).sum()
pos3 =(( df_mask['Class']==3)&(df_mask['Label']==1)).sum()
pos4 =(( df_mask['Class']==4)&(df_mask['Label']==1)).sum()
num_image = len(df_mask)//4
num = len(df_mask)
 pos =(df_mask['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<import_modules>
test_predictions=gs.predict(test_data )
import sys
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F<define_search_model>
submit=pd.DataFrame({'Id': ids, 'Target': test_predictions} )
<load_pretrained><EOS> 
submit.to_csv('submit.csv', index=False )