Kaggle competition: Quora Insincere Questions Classification
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
def save_model(model, model_path):

 torch.save(model.state_dict() , model_path)
 def load_model(model, model_path, use_cuda=False):

 map_location = 'cpu'
if use_cuda and torch.cuda.is_available() :
map_location = 'cuda:0'
model.load_state_dict(torch.load(model_path, map_location))
return model
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<prepare_x_and_y>
class TextCNN(nn.Module):

 def __init__(self, args):
super(TextCNN, self ).__init__()
vocab_size = args["vocab_size"]
pretrained_embed = args["pretrained_embed"]
padding_idx = args["padding_idx"]
num_classes = 1
kernel_nums = [100, 100, 100]
kernel_sizes = [3, 4, 5]
embed_dim = 300
hidden_dim = 100
drop_prob = 0.5
if pretrained_embed is None:
self.embed = nn.Embedding(vocab_size, embed_dim)
 else:
self.embed = nn.Embedding.from_pretrained(
 pretrained_embed, freeze=False)
 self.embed.padding_idx = padding_idx
self.convs = nn.ModuleList(
 [nn.Conv1d(embed_dim, kn, ks)
 for kn, ks in zip(kernel_nums, kernel_sizes)])
 self.fc = nn.Linear(sum(kernel_nums), hidden_dim)
 self.act = nn.ReLU()
self.drop = nn.Dropout(drop_prob)
 self.out = nn.Linear(hidden_dim, num_classes)
 self.loss = nn.BCEWithLogitsLoss()
def forward(self, word_seq):
e = self.drop(self.embed(word_seq))
e = e.transpose(1, 2)
 ps = []
for conv in self.convs:
c = conv(e)
 p = F.max_pool1d(c, kernel_size=c.size(-1)).squeeze(-1)
 ps.append(p)
 p = torch.cat(ps, dim=1)
 f = self.drop(self.act(self.fc(p)))
 logits = self.out(f ).squeeze(-1)
 return logits
class Trainer(object):

 def __init__(self, **kwargs):
self.n_epochs = kwargs["epochs"]
self.batch_size = kwargs["batch_size"]
self.validate = kwargs["validate"]
self.save_best_dev = kwargs["save_best_dev"]
self.use_cuda = kwargs["use_cuda"]
self.print_every_step = kwargs["print_every_step"]
self.optimizer = kwargs["optimizer"]
self.model_path = kwargs["model_path"]
self.eval_metrics = kwargs["eval_metrics"]
self._best_accuracy = 0.0
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def train(self, network, train_data, dev_data=None):
network = network.to(self.device)
 train_iter = torchtext.data.Iterator(
 dataset=train_data, batch_size=self.batch_size,
train=True, shuffle=True, sort=False,
device=self.device)
 if self.validate:
default_valid_args = {
"batch_size": max(8, self.batch_size // 10),
"use_cuda": self.use_cuda}
validator = Tester(**default_valid_args)
 start = time.time()
for epoch in range(1, self.n_epochs + 1):
network.train()
train_iter.init_epoch()
self._train_step(
 train_iter, network, start=start,
n_print=self.print_every_step, epoch=epoch)
 if self.validate:
if dev_data is None:
raise RuntimeError(
 "self.validate is True in trainer, "
"but dev_data is None."
" Please provide the validation data.")
 eval_results = validator.test(network, dev_data)
 if self.save_best_dev and self.best_eval_result(eval_results):
save_model(network, self.model_path)
 print("Saved better model selected by validation.")
 def _train_step(self, data_iterator, network, **kwargs):

 step = 0
for batch in data_iterator:
text, target = batch.text, batch.target
self.optimizer.zero_grad()
logits = network(text)
 loss = network.loss(logits, target.float())
 loss.backward()
self.optimizer.step()
if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0:
end = time.time()
diff = timedelta(seconds=round(end - kwargs["start"]))
print_output = "[epoch: {:>3} step: {:>4}]" \
" train loss: {:>4.6} time: {}".format(
 kwargs["epoch"], step, loss.item() , diff)
 print(print_output)
 step += 1
def best_eval_result(self, eval_results):

 assert self.eval_metrics in eval_results, \
"Evaluation doesn't contain metrics '{}'." \ 
.format(self.eval_metrics)
 accuracy = eval_results[self.eval_metrics]
if accuracy > self._best_accuracy:
self._best_accuracy = accuracy
return True
else:
return False
class Tester(object):

 def __init__(self, **kwargs):
self.batch_size = kwargs["batch_size"]
self.use_cuda = kwargs["use_cuda"]
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def test(self, network, dev_data, threshold=0.33):
network = network.to(self.device)
 network.eval()
output_list = []
truth_list = []
data_iter = torchtext.data.Iterator(
 dataset=dev_data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text, target = batch.text, batch.target
with torch.no_grad() :
prediction = network(text)
 output_list.append(prediction.detach())
 truth_list.append(target.detach())
 eval_results = self.evaluate(output_list, truth_list, threshold)
 print("[tester] {}".format(self.print_eval_results(eval_results)))
 return eval_results
def evaluate(self, predict, truth, threshold=0.33):

 y_trues, y_preds = [], []
for y_true, logit in zip(truth, predict):
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_true = y_true.cpu().numpy()
y_trues.append(y_true)
 y_preds.append(y_pred)
 y_true = np.concatenate(y_trues, axis=0)
 y_pred = np.concatenate(y_preds, axis=0)
 precision = metrics.precision_score(y_true, y_pred, pos_label=1)
 recall = metrics.recall_score(y_true, y_pred, pos_label=1)
 f1 = metrics.f1_score(y_true, y_pred, pos_label=1)
 metrics_dict = {"precision": precision, "recall": recall, "f1": f1}
return metrics_dict
def print_eval_results(self, results):

 return ", ".join(
 [str(key)+ "=" + "{:.4f}".format(value)
 for key, value in results.items() ])
 class Predictor(object):

 def __init__(self, batch_size=8, use_cuda=False):
self.batch_size = batch_size
self.use_cuda = use_cuda
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def predict(self, network, data, threshold=0.33):
network = network.to(self.device)
 network.eval()
batch_output = []
data_iter = torchtext.data.Iterator(
 dataset=data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text = batch.text
with torch.no_grad() :
prediction = network(text)
 batch_output.append(prediction.detach())
 return self._post_processor(batch_output, threshold)
 def _post_processor(self, batch_output, threshold=0.33):

 y_preds = []
for logit in batch_output:
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_preds.append(y_pred)
 y_pred = np.concatenate(y_preds, axis=0)
 return y_pred
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<create_dataframe>
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
embed_path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
submission_path = './submission.csv'
model_path = './default_model.pkl'
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, target:(prepare_image(img, cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )<create_dataframe>
def pre() :

 print("Pre-processing...")
 fix_length = 100
text = torchtext.data.Field(
 sequential=True, use_vocab=True, lower=True,
tokenize=nltk.word_tokenize, batch_first=True,
is_target=False, fix_length=fix_length)
 target = torchtext.data.Field(
 sequential=False, use_vocab=False,
batch_first=True, is_target=True)
 dataset = torchtext.data.TabularDataset(
 train_path, format='csv',
fields={"question_text":('text', text),
"target":('target', target)})
 data_test = torchtext.data.TabularDataset(
 test_path, format='csv',
fields={"question_text":('text', text)})
 text.build_vocab(dataset, data_test, min_freq=3)
 text.vocab.load_vectors(torchtext.vocab.Vectors(embed_path))
vocab_size = len(text.vocab.itos)
 padding_idx = text.vocab.stoi[text.pad_token]
data_train, data_val = dataset.split(split_ratio=0.9)
 print("train set size:", len(data_train))
print("val set size:", len(data_val))
print("test set size:", len(data_test))
print("vocab size:", len(text.vocab.itos))
print("embed shape:", text.vocab.vectors.shape)
 print('')
 args_dict = {
"data_train": data_train, "data_val": data_val,
"data_test": data_test, "vocab_size": vocab_size,
"padding_idx": padding_idx}
return args_dict
ds = get_dataset(files_test, CFG, labeled=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<choose_model_class>
args = pre()
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
def train(**args):

 print("Training...")
 data_train = args["data_train"]
pretrained_embed = data_train.fields["text"].vocab.vectors
model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": pretrained_embed,
}
model = TextCNN(model_args)
 trainer_args = {
"epochs": 8,
"batch_size": 128,
"validate": True,
"save_best_dev": True,
"use_cuda": True,
"print_every_step": 1000,
"optimizer": torch.optim.Adam(model.parameters() , lr=1e-3),
"model_path": model_path,
"eval_metrics": "f1",
}
trainer = Trainer(**trainer_args)
 data_val = args["data_val"]
trainer.train(model, data_train, dev_data=data_val)
 print('' )
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 outputs = []
for i in range(cfg['net_count']):
constructor = getattr(efn, f'EfficientNetB{i}')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.summary()
return model<choose_model_class>
train(**args )
def compile_new_model(cfg):
with strategy.scope() :
model = get_model(cfg)
 losses = [tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac'])
 for i in range(cfg['net_count'])]
model.compile(
 optimizer = cfg['optimizer'],
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model<train_model>
def test(**args):

 print("Testing...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = TextCNN(model_args)
 load_model(model, model_path, use_cuda=True)
 tester_args = {
"batch_size": 128,
"use_cuda": True,
}
tester = Tester(**tester_args)
 data_val = args["data_val"]
best_thresh, best_f1 = 0., 0.
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 f1 = tester.test(model, data_val, threshold=thresh)["f1"]
print("threshold: {:>.2f} f1: {:>.4f}".format(thresh, f1))
if f1 > best_f1:
best_thresh, best_f1 = thresh, f1
args["threshold"] = best_thresh
print("best f1 on dev: {:>.4f} threshold: {:>.2f}".format(
 best_f1, best_thresh))
print('')
 return args
<predict_on_test><EOS> 
def infer(**args):

 print("Predicting...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = TextCNN(model_args)
 load_model(model, model_path, use_cuda=True)
 predictor = Predictor(batch_size=128, use_cuda=False)
 data_test = args["data_test"]
threshold = args["threshold"]
y_pred = predictor.predict(model, data_test, threshold=threshold)
 test_df = pd.read_csv(test_path, index_col=False, header=0)
 data = {"qid": test_df["qid"], "prediction": y_pred}
subm_df = pd.DataFrame(data=data)
 subm_df.to_csv(submission_path, header=True, index=False)
 print("submission saved as {}.".format(submission_path))
print('' )