Kaggle competition: Histopathologic Cancer Detection
def get_selected_text(text: str, start_idx: int, end_idx: int, offsets: np.array)-> str:

 assert offsets.shape ==(96, 2)
 selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0] : offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1: str, str2: str)-> float:

 a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(
 text: str,
start_idx: int,
end_idx: int,
start_logits: np.array,
end_logits: np.array,
offsets: np.array, 
)-> float:

 assert start_logits.shape ==(96,)
 assert end_logits.shape ==(96,)
 assert offsets.shape ==(96, 2)
 start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
IMG_SIZE =(96, 96)
 IN_SHAPE =(*IMG_SIZE, 3)
 BATCH_SIZE = 64
def loop_through_data_loader(
 model: TweetModel,
dataloaders_dict: Dict[str, torch.utils.data.DataLoader],
optimizer: optim.AdamW,
num_epochs: int,
epoch: int,
phase: str, 
)-> TweetModel:

 assert phase in ["train", "val"]
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in dataloaders_dict[phase]:
ids = data["ids"].cuda()
masks = data["masks"].cuda()
tweet = data["tweet"]
offsets = data["offsets"].numpy()
start_idx = data["start_idx"].cuda()
end_idx = data["end_idx"].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == "train"):
start_logits, end_logits = model(ids, masks)
 loss = loss_fn(start_logits, end_logits, start_idx, end_idx)
 if phase == "train":
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i], 
)
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print(
 "Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}".format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard 
)
)
 return model<train_model>
df_data = pd.read_csv('.. /input/train_labels.csv')
 train_dir = ".. /input/train/"
test_dir = ".. /input/test"
def train_one_epoch(
 model: TweetModel,
dataloaders_dict: Dict[str, torch.utils.data.DataLoader],
optimizer: optim.AdamW,
num_epochs: int,
epoch: int, 
)-> TweetModel:

 model.train()
model = loop_through_data_loader(
 model, dataloaders_dict, optimizer, num_epochs, epoch, "train", 
)
 model.eval()
model = loop_through_data_loader(
 model, dataloaders_dict, optimizer, num_epochs, epoch, "val" 
)
 return model<train_model>
train, valid = train_test_split(df_data,test_size=0.15)
 train_datagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean())/ x.std() if x.std() > 0 else x,
horizontal_flip=True, vertical_flip=True,
rotation_range=90, shear_range=0.05, zoom_range=0.1)
 test_datagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean())/ x.std() if x.std() > 0 else x)
 train_generator = train_datagen.flow_from_dataframe(
 dataframe = train,
directory='.. /input/train/',
x_col='id',
y_col='label',
has_ext=False,
batch_size=BATCH_SIZE,
seed=2018,
shuffle=True,
class_mode='binary',
target_size=IMG_SIZE)
 valid_generator = test_datagen.flow_from_dataframe(
 dataframe = valid,
directory='.. /input/train/',
x_col='id',
y_col='label',
has_ext=False,
batch_size=BATCH_SIZE,
seed=2018,
shuffle=False,
class_mode='binary',
target_size=IMG_SIZE
)
def train_model(
 model: TweetModel,
dataloaders_dict: Dict[str, torch.utils.data.DataLoader],
optimizer: optim.AdamW,
num_epochs: int,
filename: str, 
)-> None:

 model.cuda()
for epoch in range(num_epochs):
model = train_one_epoch(model, dataloaders_dict, optimizer, num_epochs, epoch)
 torch.save(model.state_dict() , filename )<load_from_csv>
conv_base = VGG16(include_top = False,
input_shape = IN_SHAPE,
weights='imagenet')
 for layer in conv_base.layers[:-12]:
layer.trainable = False
for layer in conv_base.layers:
print(layer, layer.trainable )
%%time
if __name__ == "__main__":
if not RUN_LOCAL:
train_df = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 else:
train_df = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv", nrows=10)
 train_df["text"] = train_df["text"].astype(str)
 train_df["selected_text"] = train_df["selected_text"].astype(str)
 for fold,(train_idx, val_idx)in enumerate(
 SKF.split(train_df, train_df.sentiment), start=1 
):
print(f"Fold: {fold}")
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, BATCH_SIZE)
 train_model(
 model, dataloaders_dict, optimizer, NUM_EPOCHS, f"roberta_fold{fold}.pth",
)<load_from_csv>
model = Sequential()
model.add(conv_base)
 model.add(Flatten())
 model.add(Dense(1024, activation="relu"))
model.add(Dropout(0.6))
model.add(BatchNormalization())
 model.add(Dense(512, activation="relu"))
model.add(Dropout(0.6))
model.add(BatchNormalization())
 model.add(Dense(1, activation = "sigmoid"))
conv_base.summary()
%%time
if __name__ == "__main__":
if not RUN_LOCAL:
test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 else:
test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv", nrows=1)
 test_df["text"] = test_df["text"].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(SKF.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f"./roberta_fold{fold+1}.pth"))
model.eval()
models.append(model)
 for data in test_loader:
ids = data["ids"].cuda()
masks = data["masks"].cuda()
tweet = data["tweet"]
offsets = data["offsets"].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<load_from_csv>
model.compile(optimizers.Adam(0.001), loss = "binary_crossentropy", metrics=["accuracy"] )
if __name__ == "__main__":
if not RUN_LOCAL:
sub_df = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 else:
sub_df = pd.read_csv(
 ".. /input/tweet-sentiment-extraction/sample_submission.csv", nrows=1 
)
 sub_df["selected_text"] = predictions
sub_df["selected_text"] = sub_df["selected_text"].apply(
 lambda x: x.replace("!!!!", "!")if len(x.split())== 1 else x 
)
 sub_df["selected_text"] = sub_df["selected_text"].apply(
 lambda x: x.replace(".. ", ".")if len(x.split())== 1 else x 
)
 sub_df["selected_text"] = sub_df["selected_text"].apply(
 lambda x: x.replace("...", ".")if len(x.split())== 1 else x 
)
 sub_df.to_csv("submission.csv", index=False)
 sub_df.head()<import_modules>
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
earlystopper = EarlyStopping(monitor='val_loss', patience=3, verbose=1, restore_best_weights=True)
 learning_rate_decay = ReduceLROnPlateau(monitor='acc', patience=2, verbose=1, factor=0.3, min_lr=1e-5)
 history = model.fit_generator(train_generator, steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=50,
callbacks=[learning_rate_decay, earlystopper] )
get_linear_schedule_with_warmup,
get_cosine_with_hard_restarts_schedule_with_warmup )<define_variables>
base_test_dir = '.. /input/test/'
test_files = glob(os.path.join(base_test_dir,'*.tif'))
submission = pd.DataFrame()
file_batch = 5000
max_idx = len(test_files)
 for idx in range(0, max_idx, file_batch):
print("Indexes: %i - %i"%(idx, idx+file_batch))
test_df = pd.DataFrame({'path': test_files[idx:idx+file_batch]})
 test_df['id'] = test_df.path.map(lambda x: x.split('/')[3].split(".")[0])
 test_df['image'] = test_df['path'].map(imread)
 K_test = np.stack(test_df["image"].values)
 K_test =(K_test - K_test.mean())/ K_test.std()
predictions = model.predict(K_test)
 test_df['label'] = predictions
submission = pd.concat([submission, test_df[["id", "label"]]])
 submission.head()
<import_modules><EOS> 
submission.to_csv("submission.csv", index = False, header = True )