Kaggle competition: Understanding Clouds from Satellite Images
<SOS> metric: Dice Kaggle data source: understanding-clouds-from-satellite-images<install_modules>
!pip install tensorflow-gpu==1.14.0 --quiet
!pip install keras==2.2.4 --quiet
!pip install pyspellchecker<set_options>
!pip install tta-wrapper --quiet
seed = 0
seed_everything(seed)
 warnings.filterwarnings("ignore" )
np.random.seed(31415)
 spell = spc.SpellChecker()<categorify>
train = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 submission = pd.read_csv('.. /input/understanding_cloud_organization/sample_submission.csv')
 train['image'] = train['Image_Label'].apply(lambda x: x.split('_')[0])
 train['label'] = train['Image_Label'].apply(lambda x: x.split('_')[1])
 submission['image'] = submission['Image_Label'].apply(lambda x: x.split('_')[0])
 test = pd.DataFrame(submission['image'].unique() , columns=['image'])
 train_df = pd.pivot_table(train, index=['image'], values=['EncodedPixels'], columns=['label'], aggfunc=np.min ).reset_index()
train_df.columns = ['image', 'Fish_mask', 'Flower_mask', 'Gravel_mask', 'Sugar_mask']
print('Compete set samples:', len(train_df))
print('Test samples:', len(submission))
display(train.head() )
def add_location_to_text(row):

 if row['location'] is not np.nan:
return row['text']+" "+row['location']
return row['text']
def add_keywords_to_text(row):

 if row['keyword'] is not np.nan:
return row['text']+" "+row['keyword']
return row['text']
def add_sp1(text):

 def prepocess_text_for_spell(words):
words = list(filter(lambda x: len(x)>0, words))
return list(filter(lambda x:
not x.startswith("
x[0] != x[0].capitalize() , words))
words = prepocess_text_for_spell(text.split(" "))
return len(spell.unknown(words))
def add_wc(text):

 return len(text.split(' '))
def number_hash(text):

 words = list(filter(lambda x: len(x)>0, text.split(' ')))
 return len(list(filter(lambda x: x.startswith('
def number_of_chars(text):

 return len(text)
 def has_keyword(keyword):

 return int(not keyword is np.nan)
 def has_location(location):

 return int(not location is np.nan)
 def apply_all_feature(df):

 df.loc[:,'sp1'] = df['text'].apply(add_sp1 ).values
df.loc[:,'wc'] = df['text'].apply(add_wc ).values
df.loc[:,'hst'] = df['text'].apply(number_hash ).values
df.loc[:,'ch'] = df['text'].apply(number_of_chars ).values
df.loc[:,'loc'] = df['location'].apply(has_location ).values
df.loc[:,'text'] = df.apply(add_location_to_text,axis=1)
 df.loc[:,'text'] = df.apply(add_keywords_to_text,axis=1)
 df.loc[:,'hl'] = df.apply(has_location,axis=1)
 df.loc[:,'hk'] = df.apply(has_keyword,axis=1)
 def build_features(df_train,df_test,use_extra=True,use_nmf=False):

 Y_train = df_train['target'].values
n_train = len(Y_train)
 n_test = len(df_test)
 df_train = df_train.drop('target',axis=1)
 vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,stop_words='english',strip_accents='ascii',min_df=0.,ngram_range=(1,2))
df = df_train.append(df_test, ignore_index = True, sort=False)
 assert len(df)==n_train+n_test
df.loc[:,'text'] = df['text'].apply(lambda x: x.lower())
 X = vectorizer.fit_transform(df['text'])
 if use_extra:
apply_all_feature(df)
 X_custom = sklearn.preprocessing.scale(df[['sp1','wc','hst','ch','loc']].values,with_mean=False)
 X = scipy.sparse.hstack(( X,X_custom)).tocsr()
if use_nmf:
model = NMF(n_components=15, init='random', random_state=0)
 X = scipy.sparse.hstack(( X,model.fit_transform(X),model.fit_transform(X.log1p())) ).tocsr()
X_train = X[:n_train]
X_test = X[n_train:,:]
if 'target' in df_test.columns:
Y_test = df_test['target']
return X_train,X_test,Y_train,Y_test
else:
return X_train,X_test,Y_train,None<load_from_csv>
X_train, X_val = train_test_split(train_df, test_size=0.2, random_state=seed)
 X_train['set'] = 'train'
X_val['set'] = 'validation'
test['set'] = 'test'
print('Train samples: ', len(X_train))
print('Validation samples: ', len(X_val))
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 df_train,df_test = sklearn.model_selection.train_test_split(train,test_size=0.33)
 X_train,X_test,Y_train,Y_test = build_features(df_train,df_test,use_extra=True,use_nmf=True )<choose_model_class>
BACKBONE = 'resnet18'
BATCH_SIZE = 16
EPOCHS = 40
LEARNING_RATE = 1e-3
HEIGHT = 384
WIDTH = 480
CHANNELS = 3
N_CLASSES = 4
ES_PATIENCE = 5
RLROP_PATIENCE = 3
DECAY_DROP = 0.2
model_path = 'uNet_%s_%sx%s.h5' %(BACKBONE, HEIGHT, WIDTH )
rreg = RidgeClassifier(alpha=.8,solver='sag')
 lass = Lasso(alpha=.01)
 rf = RandomForestClassifier(max_depth=20, random_state=0, n_estimators=100)
 svm = LinearSVC(penalty='l2',max_iter=10000)
 svm = SVC(kernel='linear',max_iter=10000 )<predict_on_test>
train_generator = DataGenerator(
 directory=train_images_dest_path,
dataframe=X_train,
target_df=train,
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
augmentation=augmentation,
seed=seed)
 valid_generator = DataGenerator(
 directory=validation_images_dest_path,
dataframe=X_val,
target_df=train,
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed )
rreg.fit(X_train, Y_train)
 Y_pred = rreg.predict(X_test)
 print(sklearn.metrics.classification_report(Y_test,Y_pred))<train_model>
model = sm.Unet(backbone_name=BACKBONE,
encoder_weights='imagenet',
classes=N_CLASSES,
activation='sigmoid',
input_shape=(HEIGHT, WIDTH, CHANNELS))
checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True, save_weights_only=True)
 es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)
 metric_list = [dice_coef, sm.metrics.iou_score]
callback_list = [checkpoint, es, rlrop]
optimizer = RAdam(learning_rate=LEARNING_RATE, warmup_proportion=0.1)
 model.compile(optimizer=optimizer, loss=sm.losses.bce_dice_loss, metrics=metric_list)
 model.summary()
lass.fit(X_train, Y_train)
 Y_pred = list(map(lambda x: x>=.5, lass.predict(X_test)))
 print(sklearn.metrics.classification_report(Y_test,Y_pred))<predict_on_test>
STEP_SIZE_TRAIN = len(X_train)//BATCH_SIZE
STEP_SIZE_VALID = len(X_val)//BATCH_SIZE
history = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
callbacks=callback_list,
epochs=EPOCHS,
verbose=2 ).history
rf.fit(X_train, Y_train)
 Y_pred = rf.predict(X_test)
 print(sklearn.metrics.classification_report(Y_test,Y_pred))<predict_on_test>
class_names = ['Fish ', 'Flower', 'Gravel', 'Sugar ']
best_tresholds = [.5,.5,.5,.35]
best_masks = [25000, 20000, 22500, 15000]
for index, name in enumerate(class_names):
print('%s treshold=%.2f mask size=%d' %(name, best_tresholds[index], best_masks[index]))
svm.fit(X_train, Y_train)
 Y_pred = svm.predict(X_test)
 print(sklearn.metrics.classification_report(Y_test,Y_pred))<prepare_x_and_y>
train_metrics = get_metrics(model, train, X_train, train_images_dest_path, best_tresholds, best_masks, seed=seed, preprocessing=preprocessing, set_name='Train')
 display(train_metrics)
 validation_metrics = get_metrics(model, train, X_val, validation_images_dest_path, best_tresholds, best_masks, seed=seed, preprocessing=preprocessing, set_name='Validation')
 display(validation_metrics )
x_train,x_test,y_train,y_test = build_features(train,
test,
use_extra = True,
use_nmf = True)
 kf = KFold(n_splits=5 )<find_best_model_class>
model = tta_segmentation(model, h_flip=True, v_flip=True, h_shift=(-10, 10), v_shift=(-10, 10), merge='mean' )
F = []
A = np.round(np.linspace(0,1,11),2)
 for a in tqdm(A):
rreg = RidgeClassifier(alpha=a,solver='sag')
 f1 = []
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
kf_x_train, kf_y_train = x_train[train_index], y_train[train_index]
kf_x_val, kf_y_val = x_train[test_index], y_train[test_index]
rreg.fit(kf_x_train, kf_y_train)
 y_pred = rreg.predict(kf_x_val)
 res = sklearn.metrics.classification_report(kf_y_val,y_pred, output_dict=True)
 f1.append(res['1']['f1-score'])
 F.append(f1 )<prepare_x_and_y>
test_df = []
for i in range(0, test.shape[0], 300):
batch_idx = list(range(i, min(test.shape[0], i + 300)))
 batch_set = test[batch_idx[0]: batch_idx[-1]+1]
test_generator = DataGenerator(
 directory=test_images_dest_path,
dataframe=batch_set,
target_df=submission,
batch_size=1,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed,
mode='predict',
shuffle=False)
 preds = model.predict_generator(test_generator)
 for index, b in enumerate(batch_idx):
filename = test['image'].iloc[b]
image_df = submission[submission['image'] == filename].copy()
pred_masks = preds[index, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels'] = pred_rles
pred_masks_post = preds[index, ].astype('float32')
 for class_index in range(N_CLASSES):
pred_mask = pred_masks_post[...,class_index]
pred_mask = post_process(pred_mask, threshold=best_tresholds[class_index], min_size=best_masks[class_index])
 pred_masks_post[...,class_index] = pred_mask
pred_rles_post = build_rles(pred_masks_post, reshape=(350, 525))
image_df['EncodedPixels_post'] = pred_rles_post
test_df.append(image_df)
 sub_df = pd.concat(test_df )
X_train,X_test,Y_train,Y_test = build_features(train,test,use_extra=True,use_nmf=True )<save_to_csv>
submission_df = sub_df[['Image_Label' ,'EncodedPixels']]
submission_df.to_csv('submission.csv', index=False)
 display(submission_df.head() )
rreg = RidgeClassifier(alpha=.3,solver='sag')
 rreg.fit(X_train, Y_train)
 sub = rreg.predict(X_test)
 pd.DataFrame(np.array([test['id'].values,sub] ).T,columns=['id','target'] ).to_csv('submission.csv',sep=',',
header=True,
index=False )<set_options>
submission_df_post = sub_df[['Image_Label' ,'EncodedPixels_post']]
submission_df_post.columns = ['Image_Label' ,'EncodedPixels']
submission_df_post.to_csv('submission_post.csv', index=False)
 display(submission_df_post.head() )
<import_modules><EOS> 
if os.path.exists(train_images_dest_path):
shutil.rmtree(train_images_dest_path)
 if os.path.exists(validation_images_dest_path):
shutil.rmtree(validation_images_dest_path)
 if os.path.exists(test_images_dest_path):
shutil.rmtree(test_images_dest_path )