Kaggle competition: Understanding Clouds from Satellite Images
titanic_train=pd.read_csv(train_path)
 titanic_test=pd.read_csv(test_path)
 target=titanic_train['Survived']
Id=titanic_test[['PassengerId']]<train_model>
path = '.. /input/understanding_cloud_organization'
class NullTransformer(BaseEstimator,TransformerMixin):
def fit(self,df,y=None):
return self
def transform(self,df):
df['Embarked'].fillna('S',inplace=True)
 df['Cabin'].fillna('X',inplace=True)
 missing_Fare_index=list(df['Fare'][df['Fare'].isnull() ].index)
 for index in missing_Fare_index:
if df['Pclass'][index]==1:
df['Fare'].iloc[index]=df.groupby('Pclass')['Fare'].median() [1]
elif df['Pclass'][index]==2:
df['Fare'].iloc[index]=df.groupby('Pclass')['Fare'].median() [2]
elif df['Pclass'][index]==3:
df['Fare'].iloc[index]=df.groupby('Pclass')['Fare'].median() [3]
index_NaN_age = list(df["Age"][df["Age"].isnull() ].index)
 for index in index_NaN_age :
if df['Pclass'][index]==1:
df['Age'].iloc[index]=df.groupby('Pclass')['Age'].median() [1]
elif df['Pclass'][index]==2:
df['Age'].iloc[index]=df.groupby('Pclass')['Age'].median() [2]
elif df['Pclass'][index]==3:
df['Age'].iloc[index]=df.groupby('Pclass')['Age'].median() [3]
return df<categorify>
Train_Dir = '/kaggle/input/understanding_cloud_organization/train_images'
Test_Dir = 'kaggle/input/understanding_cloud_organization/test_images'
for img in tqdm(os.listdir(Train_Dir)) :
imgr = cv2.imread(os.path.join(Train_Dir,img))
class FeatureExtraction(BaseEstimator,TransformerMixin):
def fit(self,df,y=None):
return self
def transform(self,df):
df['Family']=df['SibSp']+df['Parch']+1
def groupfamily(x):
if x==1:
return 1
elif(2<=x)&(x<=4):
return 2
elif(5<=x)&(x<=6):
return 3
else:
return 4
df['Family']=df['Family'].apply(groupfamily)
 df.drop(['SibSp','Parch','PassengerId'],inplace=True,axis=1)
 if 'Survived' in list(df.keys()):
df.drop(['Survived'],inplace=True,axis=1)
 df['Ticket_freq']=df.groupby('Ticket')['Ticket'].transform('count')
 df.drop(['Ticket'],inplace=True,axis=1)
 df['Cabin']=df['Cabin'].str.get(i=0)
 df['Cabin'] = df['Cabin'].replace(['A', 'B', 'C','T'], 'ABCT')
 df['Cabin'] = df['Cabin'].replace(['D', 'E'], 'DE')
 df['Cabin'] = df['Cabin'].replace(['F', 'G'], 'FG')
 df['Name']=df['Name'].str.split(pat=', ',n=1,expand=True)[1].str.split(pat='.',n=1,expand=True)[0]
def Name_transform(x):
if x=='Mr':
return 'Mr'
elif x=='Mrs':
return 'Mrs'
elif x=='Miss':
return 'Miss'
else:
return 'etc'
df['Name']=df['Name'].apply(Name_transform)
 AGE_BINS=[-1,5,17,20,24,28,32,40,48,100]
df['Age_cat']=pd.cut(df['Age'],AGE_BINS,labels=[0.,1.,2.,3.,4.,5.,6.,7.,8.])
 FARE_BINS=[-1,7.35, 7.82, 8, 10, 13, 23,30, 45, 80, 150, 1000]
df['Fare_cat']=pd.cut(df['Fare'],bins=FARE_BINS,labels=[0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.])
 df.drop(['Age','Fare'],inplace=True,axis=1)
 return df<categorify>
train_csv_folder = '/kaggle/input/understanding_cloud_organization/train.csv'
train_csv = pd.read_csv(train_csv_folder)
attribs=['Pclass','Name','Sex','Age','Ticket','Fare','Cabin','Embarked','SibSp','Parch']
num_attribs=['Pclass','Age_cat','Fare_cat',"Ticket_freq"]
cat_attribs=['Name','Sex','Cabin','Embarked','Family']
pipeline1=Pipeline([ 
('NT',NullTransformer()), 
('FE',FeatureExtraction())
 ])
 train=pipeline1.fit_transform(titanic_train)
 test=pipeline1.transform(titanic_test)
 train=pd.get_dummies(train,columns=cat_attribs)
 test=pd.get_dummies(test,columns=cat_attribs)
<data_type_conversions>
train_csv['ImageId'] = train_csv['Image_Label'].apply(lambda x: x.split('_')[0])
 train_csv['ClassId'] = train_csv['Image_Label'].apply(lambda x: x.split('_')[1])
 train_csv['hasmask'] = ~train_csv['EncodedPixels'].isna()
train=train.to_numpy()
test=test.to_numpy()<choose_model_class>
mask_count_df = train_csv.groupby('ImageId' ).agg(np.sum ).reset_index()
mask_count_df.sort_values('hasmask', ascending=False, inplace=True)
 print(mask_count_df.shape)
 mask_count_df.head(10 )
kfolds=StratifiedKFold(n_splits=3)
 scores=[]
svc=SVC(random_state=42)
 rdf=RandomForestClassifier(random_state=42)
 ada=AdaBoostClassifier(random_state=42)
 log=LogisticRegression(random_state=42)
 grb=GradientBoostingClassifier(random_state=42)
 dct=DecisionTreeClassifier(random_state=42)
 ext=ExtraTreesClassifier(random_state=42)
 xgb=XGBRFClassifier(random_state=42)
 xgbc=XGBClassifier(random_state=42)
 models=[svc,rdf,ada,log,grb,dct,ext,xgb,xgbc]
for model in models:
scores.append(cross_val_score(model,train,target,cv=kfolds,n_jobs=-1))
scores_mean=np.mean(scores,axis=1 )<train_on_grid>
def rle_decode(mask_rle: str = '', shape: tuple =(1400, 2100)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape, order='F' )
def grid_search(estimator,param,X,y):
grid=GridSearchCV(estimator, param, n_jobs=-1, cv=kfolds, return_train_score=True)
 grid.fit(X,y)
 grid_results=grid.cv_results_
print('
BestParams and Score :
',grid.best_params_,'
',grid.best_score_)
 return grid.best_estimator_ , grid, estimator<train_on_grid>
def np_resize(img, input_shape):

 height, width = input_shape
return cv2.resize(img,(width, height))
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape, reshape=None):
depth = len(rles)
 if reshape is None:
masks = np.zeros(( *input_shape, depth))
else:
masks = np.zeros(( *reshape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
if reshape is None:
masks[:, :, i] = rle2mask(rle, input_shape)
 else:
mask = rle2mask(rle, input_shape)
 reshaped_mask = np_resize(mask, reshape)
 masks[:, :, i] = reshaped_mask
return masks
def build_rles(masks, reshape=None):
width, height, depth = masks.shape
rles = []
for i in range(depth):
mask = masks[:, :, i]
if reshape:
mask = mask.astype(np.float32)
 mask = np_resize(mask, reshape ).astype(np.int64)
 rle = mask2rle(mask)
 rles.append(rle)
 return rles
def random_grid_search(estimator,param_d,X,y,n):
grid=RandomizedSearchCV(estimator, param_d, n_jobs=-1, cv=kfolds, return_train_score=True, n_iter=n)
 grid.fit(X,y)
 grid_results=grid.cv_results_
print('
BestParams and Score :
',grid.best_params_,'
',grid.best_score_)
 return grid.best_estimator_ , grid, estimator<define_search_space>
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path='/kaggle/input/understanding_cloud_organization/train_images',
batch_size=32, dim=(1400, 2100), n_channels=3, reshape=None,
augment=False, n_classes=4, random_state=2019, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.reshape = reshape
self.n_channels = n_channels
self.augment = augment
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
np.random.seed(self.random_state)
 def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 if self.augment:
X, y = self.__augment_batch(X, y)
 print(X,y)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
if self.reshape is None:
X = np.empty(( self.batch_size, *self.dim, self.n_channels))
else:
X = np.empty(( self.batch_size, *self.reshape, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_rgb(img_path)
 if self.reshape is not None:
img = np_resize(img, self.reshape)
 X[i,] = img
return X
def __generate_y(self, list_IDs_batch):
if self.reshape is None:
y = np.empty(( self.batch_size, *self.dim, self.n_classes), dtype=int)
 else:
y = np.empty(( self.batch_size, *self.reshape, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
if self.reshape is not None:
masks = build_masks(rles, input_shape=self.dim, reshape=self.reshape)
 else:
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):
img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img
def __random_transform(self, img, masks):
composition = albu.Compose([
albu.HorizontalFlip() ,
albu.VerticalFlip() ,
albu.ShiftScaleRotate(rotate_limit=45, shift_limit=0.15, scale_limit=0.15)
 ])
 composed = composition(image=img, mask=masks)
 aug_img = composed['image']
aug_masks = composed['mask']
return aug_img, aug_masks
def __augment_batch(self, img_batch, masks_batch):
for i in range(img_batch.shape[0]):
img_batch[i, ], masks_batch[i, ] = self.__random_transform(
 img_batch[i, ], masks_batch[i, ])
 return img_batch, masks_batch
param_grid = [
{'C':[0.6, 0.8,1,1.15,1.2,1.23,1.4], 'kernel':['rbf'], 'gamma':[0.1] }
]
svc_best, svc_grid, svc=grid_search(SVC(random_state=42, probability=True),param_grid,X=train,y=target )<train_on_grid>
BATCH_SIZE = 4
train_idx, val_idx = train_test_split(
 mask_count_df.index, random_state=2019, test_size=0.2 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_csv,
batch_size=BATCH_SIZE,
reshape=(320, 480),
augment=True,
n_channels=3,
n_classes=4 
)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_csv,
batch_size=BATCH_SIZE,
reshape=(320, 480),
augment=False,
n_channels=3,
n_classes=4
)
param_grid = [
{'C': uniform(0.9,0.3), 'kernel':['rbf'], 'gamma':[0.1] }
]
svc_best, svc_grid, svc =random_grid_search(SVC(random_state=42, probability=True),param_grid,X=train,y=target,n=20 )<define_search_space>
!pip install -U git+https://github.com/qubvel/efficientnet
param_grid = [
{'learning_rate':[0.1], 'n_estimators':[30,50,100,200,240], 'max_depth':[2,3,4] }
]
gbc_best, gbc_grid, gbc =grid_search(GradientBoostingClassifier(random_state=42),param_grid,X=train,y=target )<train_on_grid>
!pip install segmentation-models --quiet
param_grid = [
{'learning_rate':[0.1], 'n_estimators': randint(80,110), 'max_depth':[3] }
]
gbc_best, gbc_grid, gbc =random_grid_search(GradientBoostingClassifier(random_state=42),param_grid,X=train,y=target,n=20 )<define_search_space>
from keras.layers import Dense
from keras.models import Model
from keras.optimizers import Adam, Nadam
from keras.callbacks import Callback, ModelCheckpoint
from keras.losses import binary_crossentropy
import albumentations as albu
import segmentation_models as sm
param_grid = [
{'penalty':['l1','l2'],'C':[0.1,0.8,0.9,1,1.1,1.2,1.3,5], 'tol':[1e-4,1e-3],'solver':['liblinear']}
]
log_best, log_grid, log =grid_search(LogisticRegression(random_state=42),param_grid,X=train,y=target )<train_on_grid>
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )
param_grid = [
{'penalty':['l1'],'C': uniform(0.7,0.3), 'tol':[1e-4],'solver':['liblinear']}
]
log_best, log_grid, log =random_grid_search(LogisticRegression(random_state=42),param_grid,X=train,y=target,n=20 )<train_on_grid>
model = sm.Unet(
 'resnet18',
classes=4,
input_shape=(320, 480, 3),
activation='sigmoid' 
)
 model.compile(optimizer=Nadam(lr=0.0002), loss=bce_dice_loss, metrics=[dice_coef])
 model.summary()
param_grid = [
{'n_estimators':randint(20,60),'max_depth':[4], 'gamma':[0.1]}
]
xgb_best, xgb_grid, xgb =random_grid_search(XGBRFClassifier(random_state=42),param_grid,X=train,y=target,n=20 )<train_on_grid>
sub_df = pd.read_csv('/kaggle/input/understanding_cloud_organization/sample_submission.csv')
 sub_df['ImageId'] = sub_df['Image_Label'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['ImageId'].unique() , columns=['ImageId'] )
param_grid = [
{'n_estimators': randint(70,120),'max_depth':[3], 'gamma':[0.1]}
]
xgbc_best, xgbc_grid, xgbc =random_grid_search(XGBClassifier(random_state=42),param_grid,X=train,y=target,n=20 )<define_search_space>
resnetpath = '/kaggle/input/resnet18/modelefficient.h5'
model.load_weights(resnetpath)
 test_df = []
for i in range(0, test_imgs.shape[0], 500):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 500)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
dim=(350, 525),
reshape=(320, 480),
n_channels=3,
base_path='/kaggle/input/understanding_cloud_organization/test_images',
target_df=sub_df,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model.predict_generator(
 test_generator,
workers=1,
verbose=1 
)
 for j, b in enumerate(batch_idx):
filename = test_imgs['ImageId'].iloc[b]
image_df = sub_df[sub_df['ImageId'] == filename].copy()
pred_masks = batch_pred_masks[j, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels'] = pred_rles
test_df.append(image_df )
param_grid = [
{'n_estimators': [30,50,100,200,300],'learning_rate':[0.9,1.0,1.1,1.2,1.3]}
]
ada_best, ada_grid, ada =grid_search(AdaBoostClassifier(random_state=42),param_grid,X=train,y=target )<train_on_grid>
os.listdir('/kaggle/input/csvfiledl/SubmissionDLearningFinal.csv' )
param_grid = [
{'n_estimators': randint(20,50),'learning_rate':[1.3]}
]
ada_best, ada_grid, ada =random_grid_search(AdaBoostClassifier(random_state=42),param_grid,X=train,y=target,n=20 )<define_search_space>
df = pd.read_csv('/kaggle/input/csvfiledl/SubmissionDLearningFinal.csv' )
<train_on_grid><EOS> 
df.to_csv('FinaldeepLearning.csv',index = False )