Kaggle competition: Korean Gender Bias Detection
opt = OptimizedRounder()
tst_pred = opt.predict(preds_A * 0.5 + preds_B * 0.5,coef=[0.5, 1.5, 2.5, 3.5])
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )<feature_engineering>
train_df = pd.read_csv('/kaggle/input/koreangenderbiasdetection/train.gender_bias.binary.csv')
 train_df.head()
t_start = time.time()<define_variables>
dev_df = pd.read_csv('/kaggle/input/koreangenderbiasdetection/dev.gender_bias.binary.csv')
 dev_df.head()
IMG_WIDTH = 512
IMG_HEIGHT = 512
CHANNEL = 3
BATCH_SIZE = 4
OLD_DATA_EPOCHS = 6
NEW_DATA_EPOCHS = 6
WARMUP_EPOCHS = 3
NUM_CLASSES = 5
SEED = 2
n_folds = 1
ES_PATIENCE = 5
RLROP_PATIENCE = 2
DECAY_DROP = 0.2<train_model>
train_df['label'] = train_df['label'].map(lambda x: 1 if x else 0)
 dev_df['label'] = dev_df['label'].map(lambda x: 1 if x else 0 )
def append_file_ext_png(file_name):
return file_name + ".png"
def append_file_ext_jpeg(file_name):
return file_name + ".jpeg"<define_variables>
train_sentences = [list(d)for d in train_df['comments'].to_list() ]
dev_sentences = [list(d)for d in dev_df['comments'].to_list() ]
BASE_DIR = '/kaggle/input/aptos2019-blindness-detection/'
TRAIN_DIR = '/kaggle/input/diabetic-retinopathy-resized/resized_train/resized_train'
TEST_DIR = '/kaggle/input/aptos2019-blindness-detection/test_images'<prepare_x_and_y>
train_labels = train_df['label'].to_list()
dev_labels = dev_df['label'].to_list()
TEST_DF = pd.read_csv(BASE_DIR + "test.csv",dtype='object')
 TRAIN_DF = pd.read_csv("/kaggle/input/diabetic-retinopathy-resized/trainLabels.csv",dtype='object')
 TRAIN_DF.columns = ['id_code', 'diagnosis']
X_COL='id_code'
Y_COL='diagnosis'
TRAIN_DF[X_COL] = TRAIN_DF[X_COL].apply(append_file_ext_jpeg)
 TEST_DF[X_COL] = TEST_DF[X_COL].apply(append_file_ext_png )<concatenate>
def make_dictionary(sentences, vocabulary_size=None, initial_words=['<UNK>', '<PAD>', '<SOS>', '<EOS>']):

 counter = Counter()
for words in sentences:
counter.update(words)
 if vocabulary_size is None:
vocabulary_size = len(counter.keys())
 vocab_words = counter.most_common(vocabulary_size)
 for initial_word in initial_words:
vocab_words.insert(0,(initial_word, 0))
word2idx = {word:idx for idx,(word, count)in enumerate(vocab_words)}
idx2word = {idx:word for word, idx in word2idx.items() }
return word2idx, idx2word
def process_sentences(sentences, word2idx, sentence_length=20, padding='<PAD>'):

 sentences_processed = []
for sentence in sentences:
if len(sentence)> sentence_length:
fixed_sentence = sentence[:sentence_length]
else:
fixed_sentence = sentence + [padding]*(sentence_length - len(sentence))
sentence_idx = [word2idx[word] if word in word2idx.keys() else word2idx['<UNK>'] for word in fixed_sentence]
sentences_processed.append(sentence_idx)
 return sentences_processed
def make_mask(sentences, sentence_length):
masks = []
for sentence in sentences:
words_count = len(sentence[:sentence_length])
 sentence_mask = np.concatenate([np.ones(words_count-1), np.ones(1), np.zeros(sentence_length-words_count)])
 masks.append(sentence_mask)
 mask = np.array(masks)
 return mask
df0 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '0']
df1 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '1']
df2 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '2']
df3 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '3']
df4 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '4']
df0 = df0.head(2000)
 df1 = df1.head(2000)
 df2 = df2.head(2000)
 TRAIN_DF = df0.append([df1, df2, df3, df4],ignore_index = True)
 TRAIN_DF = shuffle(TRAIN_DF )<choose_model_class>
word2idx, idx2word = make_dictionary(train_sentences, initial_words=['<UNK>', '<PAD>'] )
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<choose_model_class>
SENTENCE_LENGTH = 150
train_sentences_processed = process_sentences(train_sentences, word2idx, sentence_length=SENTENCE_LENGTH)
 dev_sentences_processed = process_sentences(dev_sentences, word2idx, sentence_length=SENTENCE_LENGTH )
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = EfficientNetB5(weights=None,
include_top=False,
input_tensor=input_tensor)
 base_model.load_weights('/kaggle/input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(2048, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model<choose_model_class>
train_mask = make_mask(train_sentences, sentence_length=SENTENCE_LENGTH)
 dev_mask = make_mask(dev_sentences, sentence_length=SENTENCE_LENGTH )
es = EarlyStopping(monitor='val_loss',
mode='min',
patience=ES_PATIENCE,
restore_best_weights=True,
verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss',
mode='min',
patience=RLROP_PATIENCE,
factor=DECAY_DROP,
verbose=1)
 model_checkpoint = ModelCheckpoint('EfficientNetB5.h5',
verbose=1,
save_best_only=True)
 callback_list = [es, rlrop, model_checkpoint]<create_dataframe>
import torch
from torch import nn
from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable
from torch.optim import Adam
datagen = ImageDataGenerator(
 rescale=1./255.,
validation_split=0.25)
 train_generator = datagen.flow_from_dataframe(
 dataframe=TRAIN_DF,
directory=TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="training",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
valid_generator=datagen.flow_from_dataframe(
 dataframe=TRAIN_DF,
directory=TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="validation",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size<choose_model_class>
class DatasetLoader(Dataset):
def __init__(self, sentences_processed, labels):
assert len(sentences_processed)== len(labels)
 self.sentences_processed = sentences_processed
self.labels = torch.LongTensor(labels)
 def __getitem__(self, index):
return torch.LongTensor(self.sentences_processed[index]), self.labels[index]
def __len__(self):
return len(self.sentences_processed )
model = create_model(input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNEL), n_out=NUM_CLASSES)
 for layer in model.layers:
layer.trainable = False
for i in range(-10, 0):
model.layers[i].trainable = True
metric_list = ["accuracy"]
optimizer = RAdam(lr=0.00005)
 model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=metric_list )<train_model>
train_dataset = DatasetLoader(train_sentences_processed, train_labels)
 dev_dataset = DatasetLoader(dev_sentences_processed, dev_labels )
history_warmup = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=WARMUP_EPOCHS,
verbose=1 ).history<train_model>
train_dataloader = DataLoader(train_dataset, batch_size=128)
 dev_dataloader = DataLoader(dev_dataset, batch_size=256 )
for layer in model.layers:
layer.trainable = True
optimizer = RAdam(lr=0.00005)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=metric_list)
 gc.collect()
history_finetunning = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=OLD_DATA_EPOCHS,
callbacks=callback_list,
verbose=1 ).history<load_from_csv>
import torch
import torch.nn as nn
import torch.nn.functional as F
NEW_TRAIN_DIR = '/kaggle/input/aptos2019-blindness-detection/train_images'
NEW_TRAIN_DF = pd.read_csv(BASE_DIR + "train.csv",dtype='object')
 NEW_TRAIN_DF[X_COL] = NEW_TRAIN_DF[X_COL].apply(append_file_ext_png)
 NEW_TRAIN_DF.head()<create_dataframe>
class CharCNN(nn.Module):
def __init__(self, vocab_size, embedding_size, output_size, kernel_num, kernel_sizes):
super().__init__()
self.embedding = nn.Embedding(
 vocab_size, embedding_size, padding_idx=0 
)
 self.convs = nn.ModuleList([
nn.Conv1d(embedding_size, kernel_num, kernel_size=kernel_size)
 for kernel_size in kernel_sizes
])
 self.maxpools = nn.ModuleList([
nn.MaxPool1d(kernel_size)
 for kernel_size in kernel_sizes
])
 self.linear = nn.Linear(1140, output_size)
 self.softmax = nn.LogSoftmax(dim=1)
 self.sigmoid = nn.Sigmoid()
self.dropout = nn.Dropout(p=0.5)
 def forward(self, x):
batch_size = x.size(0)
 embedded = self.embedding(x)
 embedded = embedded.transpose(1,2)
 pools = []
for conv, maxpool in zip(self.convs, self.maxpools):
feature_map = conv(embedded)
 pooled = maxpool(feature_map)
 pools.append(pooled)
 conv_concat = torch.cat(pools, dim=-1 ).view(batch_size, -1)
 conv_concat = self.dropout(conv_concat)
 logits = self.linear(conv_concat)
 return self.softmax(logits )
datagen = ImageDataGenerator(
 rescale=1./255.,
validation_split=0.25)
 train_generator = datagen.flow_from_dataframe(
 dataframe=NEW_TRAIN_DF,
directory=NEW_TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="training",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
valid_generator=datagen.flow_from_dataframe(
 dataframe=NEW_TRAIN_DF,
directory=NEW_TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="validation",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size<train_model>
model = CharCNN(
 vocab_size=len(word2idx),
embedding_size=300,
output_size=2,
kernel_num=10,
kernel_sizes=[3,4,5]
)
gc.collect()
history_finetunning = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=NEW_DATA_EPOCHS,
callbacks=callback_list,
verbose=1 ).history<choose_model_class>
if torch.cuda.is_available() :
model.to('cuda' )
optimizer = RAdam(lr=0.00005)
 metric_list = ["accuracy"]
model = load_model('EfficientNetB5.h5', compile=False)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=metric_list )<create_dataframe>
loss_function = nn.NLLLoss()
test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=TEST_DF,
directory=TEST_DIR,
x_col=X_COL,
y_col=None,
batch_size=BATCH_SIZE,
seed=SEED,
shuffle=False,
class_mode=None,
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))<define_variables>
optimizer = Adam(model.parameters() )
if test_generator.n%BATCH_SIZE > 0:
PREDICTION_STEPS =(test_generator.n//BATCH_SIZE)+ 1
else:
PREDICTION_STEPS =(test_generator.n//BATCH_SIZE)
 print(PREDICTION_STEPS )<predict_on_test>
for epoch in range(6):
losses = []
for i,(batch_data, batch_label)in enumerate(train_dataloader):
batch_data, batch_label = Variable(batch_data), Variable(batch_label)
 if torch.cuda.is_available() :
batch_data = batch_data.to('cuda')
 batch_label = batch_label.to('cuda')
 log_probs = model(batch_data)
 loss = loss_function(log_probs, batch_label.transpose(0, -1))
model.zero_grad()
loss.backward()
optimizer.step()
losses.append(loss.item())
 corrects = log_probs.data.cpu().numpy().argmax(axis=1)== batch_label.data.cpu().numpy()
train_accuracy = corrects.astype(np.long ).mean()
val_losses = []
val_accuracies = []
for test_data, test_label in dev_dataloader:
test_data, test_label = Variable(test_data), Variable(test_label)
 if torch.cuda.is_available() :
test_data = test_data.to('cuda')
 test_label = test_label.to('cuda')
 val_log_probs = model(test_data)
 val_loss = loss_function(val_log_probs, test_label)
 val_corrects = val_log_probs.data.cpu().numpy().argmax(axis=1)== test_label.data.cpu().numpy()
val_accuracy = val_corrects.astype(np.float64 ).mean()
val_losses.append(val_loss.item())
 val_accuracies.append(val_accuracy)
 message = "Epoch: {epoch:<5d} Iteration: {iteration:<5d} Loss: {loss:<.3} Val Loss: {val_loss:<.3} Train Accuracy: {train_acc:<.3} Test Accuracy: {test_acc:<.3}".format(
 epoch=epoch,
iteration= i,
loss=sum(losses)/len(losses),
val_loss=sum(val_losses)/len(val_losses),
train_acc=train_accuracy,
test_acc=sum(val_accuracies)/len(val_accuracies)
)
 print(message )
test_generator.reset()
preds = model.predict_generator(test_generator,
steps=PREDICTION_STEPS,
verbose=1)
 predictions = [np.argmax(pred)for pred in preds]<save_to_csv>
test_df = pd.read_csv('/kaggle/input/koreangenderbiasdetection/test.gender_bias.no_label.csv' )
filenames = test_generator.filenames
results = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])
 results.astype({'diagnosis': 'int64'})
 results.to_csv('submission.csv',index=False)
 print(results.head(10))<train_model>
test_sentences = [list(d)for d in test_df['comments'].to_list() ]
t_finish = time.time()
total_time = round(( t_finish-t_start)/ 3600, 4)
 print('Kernel runtime = {} hours({} minutes)'.format(total_time, int(total_time*60)) )<set_options>
test_sentences_processed = process_sentences(test_sentences, word2idx, sentence_length=SENTENCE_LENGTH)
 test_mask = make_mask(test_sentences, sentence_length=SENTENCE_LENGTH)
 test_dataset = DatasetLoader(test_sentences_processed, [0 for i in range(len(test_sentences_processed)) ])
 test_dataloader = DataLoader(test_dataset, batch_size=1 )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
test_df['label'] = res
from fastai import *
from fastai.vision import *
import torch<define_variables>
test_df['label'].value_counts()
path = Path('.. /input/aptos2019-blindness-detection')
 path_train = path/'train_images'
path_test = path/'test_images'
path, path_train, path_test<load_from_csv>
test_df['label'] = test_df['label'].map(lambda x: True if x else False )
<feature_engineering><EOS> 
test_df.to_csv('/kaggle/working/prediction.csv', index=None, header=True )