Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
<SOS> metric: LogLoss Kaggle data source: statoil-iceberg-classifier-challenge<train_on_grid>
%matplotlib inline
clf = DisasterDetector(bert_layer, max_seq_length=128, lr=0.0001, epochs=10, batch_size=32)
 clf.train(df_train )<save_to_csv>
def load_train_data() :
data = pd.read_json('.. /input/train.json')
 return data
def load_test_data() :
data = pd.read_json('.. /input/test.json')
 return data
def denoise(x, weight, multichannel):
return np.asarray([denoise_tv_chambolle(item, weight=weight, multichannel=multichannel)for item in x])
 def gaussian_blur(x, sigma, multichannel):
return np.asarray([gaussian(item, sigma=sigma, multichannel=multichannel)for item in x])
 def color_composite(data):
rgb_arrays = []
for i, row in data.iterrows() :
band_1 = np.array(row['band_1'] ).reshape(75, 75)
 band_2 = np.array(row['band_2'] ).reshape(75, 75)
 band_3 = band_1 / band_2
r =(band_1 + abs(band_1.min())) / np.max(( band_1 + abs(band_1.min())))
 g =(band_2 + abs(band_2.min())) / np.max(( band_2 + abs(band_2.min())))
 b =(band_3 + abs(band_3.min())) / np.max(( band_3 + abs(band_3.min())))
 rgb = np.dstack(( r, g, b))
rgb_arrays.append(rgb)
 return np.array(rgb_arrays )
y_pred = clf.predict(df_test)
 model_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 model_submission['target'] = np.round(y_pred ).astype('int')
 model_submission.to_csv('submission.csv', index=False)
 model_submission.describe()<load_from_csv>
def lee_filter(img, size):
img_mean = uniform_filter(img,(size, size))
img_sqr_mean = uniform_filter(img**2,(size, size))
img_variance = img_sqr_mean - img_mean**2
overall_variance = variance(img)
 img_weights = img_variance**2 /(img_variance**2 + overall_variance**2)
 img_output = img_mean + img_weights *(img - img_mean)
 return img_output
train_tweets = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_tweets = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 nlp = spacy.load('en_core_web_lg')
 train_tweets.head()<define_variables>
train_df = load_train_data()
test_df = load_test_data()
train_data = train_tweets
with nlp.disable_pipes() :
doc_vectors = np.array([nlp(text ).vector for text in train_data.text])
 test_data = test_tweets
with nlp.disable_pipes() :
test_doc_vectors = np.array([nlp(text ).vector for text in test_data.text] )<split>
label = np.array(train_df["is_iceberg"])
 train_angle = train_df['inc_angle']
train_angle = train_angle.replace('na',np.nan)
 train_angle = train_angle.astype(float ).fillna(np.mean(train_angle))
test_angle = test_df['inc_angle']
test_angle = test_angle.replace('na',np.nan)
 test_angle = test_angle.astype(float ).fillna(0)
 print('Done' )
X_train, X_test, y_train, y_test = train_test_split(doc_vectors, train_data.target,
test_size=0.2, random_state=1 )<define_search_space>
def create_dataset(data, labeled, weight_gray=0.05, weight_rgb=0.05):
band_1, band_2, images = data['band_1'].values, data['band_2'], color_composite(data)
 to_arr = lambda x: np.asarray([np.asarray(item)for item in x])
 band_1 = to_arr(band_1)
 band_2 = to_arr(band_2)
 band_3 =(band_1 / band_2)
 to_rgb = lambda x: np.asarray([(item + abs(item.min())) / np.max(( item + abs(item.min())))for item in x])
 band_1 = to_rgb(band_1)
 band_2 = to_rgb(band_2)
 band_3 = to_rgb(band_3)
 gray_reshape = lambda x: np.asarray([item.reshape(75, 75)for item in x])
 band_1 = gray_reshape(band_1)
 band_2 = gray_reshape(band_2)
 band_3 = gray_reshape(band_3)
 print('denoising')
 tf_reshape = lambda x: np.asarray([item.reshape(75, 75, 1)for item in x])
 band_1 = tf_reshape(band_1)
 band_2 = tf_reshape(band_2)
 band_3 = tf_reshape(band_3)
 img_reshape = lambda x: np.asarray([item.reshape(75,75,3)for item in x])
 images = img_reshape(images)
 band = np.concatenate([band_1, band_2, band_3], axis=3)
 x_angle = data.inc_angle
x_angle = x_angle.replace('na',np.nan)
 x_angle = x_angle.astype(float ).fillna(np.mean(x_angle))
x_angle = np.array(x_angle)
 if labeled:
y = np.array(data['is_iceberg'])
 else:
y = None
print('Done')
 return y, x_angle, band, images
estimators_list = [50, 100, 150, 200, 250]
learning_rate_list = [0.3, 0.4, 0.5, 0.6, 0.7]
XGB_estimators_list = [500, 1000, 1500, 2000, 2500]<compute_train_metric>
y, x_angle, x_band, x_img = create_dataset(train_df, True )
def SVM_score_dataset() :
model = LinearSVC(random_state=0, dual=False, max_iter=10000)
 model.fit(X_train, y_train)
 preds = model.predict(X_test)
 f1score = f1_score(y_test, preds)
 return(f1score)
 SVM_score_dataset()<train_on_grid>
np.amax(x_band )
def RF_score_dataset(n_estimators):
model = RandomForestClassifier(n_estimators=n_estimators, random_state=0)
 model.fit(X_train, y_train)
 preds = model.predict(X_test)
 f1score = f1_score(y_test, preds)
 return(f1score)
 RF_f1scores = []
for i in estimators_list:
f1 = RF_score_dataset(i)
 RF_f1scores.append(f1)
 print('Max f1 is %f when n_estimators is %s.' %(max(RF_f1scores), estimators_list[RF_f1scores.index(max(RF_f1scores)) ]))
print(max(RF_f1scores))<train_model>
x_train, x_val, \
x_angle_train, x_angle_val, \
y_train, y_val = train_test_split(x_img,x_angle, y, random_state=666, test_size=0.25)
 print('img', x_train.shape, y_train.shape)
 print('angle', x_angle_train.shape,y_val.shape)
my_model = XGBClassifier(n_estimators=1000, learning_rate=0.4)
 my_model.fit(X_train, y_train)
 preds = my_model.predict(test_doc_vectors )<save_to_csv>
adamax01 = Adamax(lr=0.0003 )
output = pd.DataFrame({'id': test_tweets.id,
'target': preds})
 output.to_csv('submission.csv', index=False )<set_options>
def model2(kernel_size=3, filters_1=32, filters_2=32, filters_3=64, filters_4=128,
optimizers=adamax01,init='lecun_normal',relu_type='selu'):
band_input = Input(shape=(75, 75, 3))
cnn = BatchNormalization()(band_input)
 for i in range(1):
cnn = Conv2D(filters=filters_1, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 for i in range(1):
cnn = Conv2D(filters=filters_2, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 for i in range(1):
cnn = Conv2D(filters=filters_3, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 for i in range(1):
cnn = Conv2D(filters=filters_4, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 cnn = Flatten()(cnn)
 cnn = Dense(512,activation= relu_type )(cnn)
 cnn = Dense(1, activation = 'sigmoid' )(cnn)
 simple_cnn = Model(inputs=[band_input],outputs=cnn)
 simple_cnn.compile(optimizer=optimizers, loss = 'binary_crossentropy', metrics = ['accuracy'])
 return simple_cnn
m = model2()
m.summary()
warnings.filterwarnings('ignore')
<define_variables>
gen = ImageDataGenerator(horizontal_flip = True,
vertical_flip = True,
width_shift_range = 0,
height_shift_range = 0,
zoom_range = 0.2,
rotation_range = 20)
 def gen_flow_for_one_inputs(X1, y):
genX1 = gen.flow(X1,y, batch_size=64,seed=1)
 while True:
X1i = genX1.next()
yield X1i[0], X1i[1]
gen_flow = gen_flow_for_one_inputs(x_train, y_train)
 gen_val = gen_flow_for_one_inputs(x_val, y_val)
 model = model2()
ss = model.fit_generator(gen_flow, validation_data=(x_val,y_val),
steps_per_epoch=len(x_train)/ 64,
epochs=130,
callbacks = callbacks_list,
verbose = 1)
class Config() :
def __init__(self):
self.work_dir = Path("/kaggle")
 self.train_path = self.work_dir / "input/nlp-getting-started/train.csv"
self.test_path = self.work_dir / "input/nlp-getting-started/test.csv"
self.submission_path = self.work_dir / "working/submission.csv"
self.model_name = "distilbert-base-uncased"
self.metric_name = "val_f1"
self.seed = 41
self.batch_size = 64
self.nfold = 5<categorify>
model.load_weights(filepath=filepath)
 score = model.evaluate(x_train, y_train,verbose=1)
 print('Train loss:', score[0])
 print('Train accuracy:', score[1])
 test_score = model.evaluate(x_val, y_val,verbose=1)
 print('Test loss:', test_score[0])
 print('Test accuracy:', test_score[1])
class DisasterTweetsDataset(Dataset):
def __init__(self, encodings, labels=None):
self.encodings = encodings
self.labels = labels
def __len__(self):
return len(self.encodings.input_ids)
 def __getitem__(self, index):
input_id = torch.tensor(self.encodings.input_ids[index])
 attention_mask = torch.tensor(self.encodings.attention_mask[index])
 label = self.labels[index] if self.labels is not None else -1
return input_id, attention_mask, label
class DisasterTweetsDataModule(pl.LightningDataModule):
def __init__(self, config):
super().__init__()
self.config = config
self.train = pd.read_csv(config.train_path)
 self.test = pd.read_csv(config.test_path)
 self.tokenizer = DistilBertTokenizer.from_pretrained(config.model_name)
 skf = StratifiedKFold(n_splits=config.nfold, shuffle=True, random_state=config.seed)
 self.fold_indices = {fold:(t_idx, v_idx)for fold,(t_idx, v_idx)in enumerate(skf.split(list(self.train.text), list(self.train.target)))}
def _fold_data(self, fold):
train_index, val_index = self.fold_indices[fold]
t_df = self.train.loc[train_index, ["text", "target"]]
v_df = self.train.loc[val_index, ["text", "target"]]
return list(t_df.text), list(v_df.text), list(t_df.target), list(v_df.target)
 def setup(self, stage=None, fold=0, verify_train_data=False):
train_texts, val_texts, train_targets, val_targets = self._fold_data(fold)
 self.train_ds = DisasterTweetsDataset(self.tokenizer(train_texts, truncation=True, padding=True), train_targets)
 self.val_ds = DisasterTweetsDataset(self.tokenizer(val_texts, truncation=True, padding=True), val_targets)
 test_texts = list(self.test.text)if not verify_train_data else list(self.train.text)
 self.test_ds = DisasterTweetsDataset(self.tokenizer(test_texts, truncation=True, padding=True))
def train_dataloader(self):
return DataLoader(self.train_ds, batch_size=self.config.batch_size)
 def val_dataloader(self):
return DataLoader(self.val_ds, batch_size=self.config.batch_size)
 def test_dataloader(self):
return DataLoader(self.test_ds, batch_size=self.config.batch_size )<load_pretrained>
model.load_weights(filepath=filepath)
class DisasterTweetsNet(pl.LightningModule):
def __init__(self, train=True):
super(DisasterTweetsNet, self ).__init__()
self.model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased", return_dict=True)
 self.F1 = F1(num_classes=2)
 if train:
self.model.train()
def forward(self, input_ids, attention_mask):
return self.model(input_ids=input_ids, attention_mask=attention_mask)
 def training_step(self, batch, batch_idx):
return self.common_step(batch, mode="train")
 def validation_step(self, batch, batch_idx):
return self.common_step(batch, mode="val")
 def test_step(self, batch, batch_idx):
input_ids, attention_masks, labels = batch
outputs = self.forward(input_ids, attention_masks)
 return torch.softmax(outputs.logits, dim=1)
 def common_step(self, batch, mode="train"):
input_ids, attention_masks, labels = batch
outputs = self.forward(input_ids, attention_masks)
 loss = F.cross_entropy(outputs.logits, labels)
 preds = torch.argmax(outputs.logits, dim=1)
 f1 = self.F1(preds, labels)
 self.log(f"{mode}_loss", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True)
 self.log(f"{mode}_f1", f1, on_step=False, on_epoch=True, prog_bar=True, logger=True)
 return loss
def configure_optimizers(self):
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{"params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay)], "weight_decay": 0.01},
{"params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0}
]
return AdamW(optimizer_grouped_parameters, lr=1e-5 )<define_variables>
q, test_angle, test_band, test_img = create_dataset(test_df, False )
class OutputRecorder(Callback):
def __init__(self, config):
super().__init__()
self.__outputs_one_epoch = []
self.__records = []
self.config = config
@property
def outputs(self):
outputs = torch.zeros(( len(self.__records), *self.__records[0].shape), dtype=self.__records[0].dtype)
 for i, ot in enumerate(self.__records):
outputs[i] = ot
return outputs
def _record(self, outputs):
outputs = outputs.cpu().detach()
self.__outputs_one_epoch.append(outputs)
 def on_test_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx):
self._record(outputs)
 def on_test_epoch_end(self, trainer, pl_module):
outputs = torch.cat(self.__outputs_one_epoch)
 self.__records.append(outputs)
 self.__outputs_one_epoch = []
def make_submission(self):
results = torch.mean(self.outputs, dim=0)
 results = torch.argmax(results, dim=1)
 test = pd.read_csv(self.config.test_path)
 test["target"] = results
test = test.loc[:, ["id", "target"]]
test.to_csv(self.config.submission_path, index=False)
 return test<choose_model_class>
test_pred = model.predict([test_img])
 test_pred = test_pred.reshape(-1)
 print('Done' )
class ModelSaveDir() :
def __init__(self, root_models_dir, metric_name="val_acc"):
self.root_models_dir = Path(root_models_dir)
 self.metric_name = metric_name
self.pattern = re.compile(r'vfold=(\d+)-epoch=(\d+)-' + metric_name + r'=(\d\.\d+)')
 def get_model_path_template(self, fold):
return self.root_models_dir /(f"vfold={fold}-" + "{epoch}-{" + self.metric_name + ":.4f}")
 def get_model_paths(self):
model_paths = {}
re_paths = [(self.pattern.search(str(p)) , str(p)) for p in self.root_models_dir.iterdir() if p.is_file() and p.suffix == ".ckpt"]
for fold in set([int(r.group(1)) for r,_ in re_paths]):
paths = [p for r,p in re_paths if int(r.group(1)) == fold]
metrics = np.array([float(r.group(3)) for r,_ in re_paths if int(r.group(1)) == fold])
 model_paths[fold] = paths[np.argmax(metrics)]
return model_paths
@classmethod
def make_new_dir(cls, work_dir, metric_name="val_acc"):
now = datetime.datetime.today()
root_models_dir = Path(work_dir)/ f"models/{now}"
root_models_dir.mkdir(parents=True)
 return ModelSaveDir(root_models_dir, metric_name )<import_modules>
img_indicies = np.random.choice(range(len(test_img)) , len(x_train)*1, False)
 test_sub = test_img[img_indicies]
test_pred_sub = test_pred[img_indicies]
Y = np.concatenate(( y_train, test_pred_sub))
print('Done' )
class CVTrainer() :
def __init__(self, config, dm=None):
self.config = config
self.dm = DisasterTweetsDataModule(self.config)if not dm else dm
def fit(self, nepoch, model_save_dir, folds=[0]):
model_paths = model_save_dir.get_model_paths()
for fold in folds:
model = DisasterTweetsNet.load_from_checkpoint(checkpoint_path=str(model_paths[fold])) if fold in model_paths else DisasterTweetsNet()
template_path = model_save_dir.get_model_path_template(fold)
 checkpoint_callback = ModelCheckpoint(filepath=template_path, monitor=self.config.metric_name, save_top_k=1, mode="max")
 logger = TensorBoardLogger(model_save_dir.root_models_dir, name=f"fold={fold}")
 self.dm.setup(fold=fold)
 trainer = pl.Trainer(gpus=1, max_epochs=nepoch, progress_bar_refresh_rate=10, checkpoint_callback=checkpoint_callback, logger=logger)
 trainer.fit(model, self.dm)
 def test(self, model_save_dir, nepoch=1, verify_train_data=False):
outputRecorder = OutputRecorder(self.config)
 model_paths = model_save_dir.get_model_paths()
for fold in model_paths.keys() :
model = DisasterTweetsNet.load_from_checkpoint(checkpoint_path=str(model_paths[fold]))
self.dm.setup(verify_train_data=verify_train_data)
 trainer = pl.Trainer(gpus=1, max_epochs=1, progress_bar_refresh_rate=10, callbacks=[outputRecorder])
 for epoch in range(nepoch):
trainer.test(model, datamodule=self.dm)
 return outputRecorder<set_options>
mm = model.fit_generator(gen_flow, validation_data=(x_val, y_val),
steps_per_epoch=len(X)/ 32,
epochs=10,
callbacks = callbacks_list,
verbose = 1,
shuffle=True )
config = Config()<define_variables>
model.load_weights(filepath=filepath)
 test_score = model.evaluate(x_val, y_val,verbose=1)
 print('Test loss:', test_score[0])
 print('Test accuracy:', test_score[1] )
model_save_dir = ModelSaveDir.make_new_dir(config.work_dir, metric_name=config.metric_name)
 model_save_dir.root_models_dir<define_variables>
model.load_weights(filepath=filepath)
 test_predictions = model.predict([test_img])
 pred_df = test_df[['id']].copy()
pred_df['is_iceberg'] = test_predictions
pred_df.to_csv('predictions.csv', index = False)
 pred_df.head(3 )