Kaggle competition: Human Protein Atlas - Single Cell Classification
<SOS> metric: OpenImagesObjDetectionSegmentationAP Kaggle data source: human-protein-atlas-single-cell-classification<load_pretrained>
!pip install ".. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl"
!pip install ".. /input/hpapytorchzoozip/pytorch_zoo-master"
!pip install ".. /input/hpacellsegmentatorraman/HPA-Cell-Segmentation/"
!pip install ".. /input/efficientnet-keras-source-code/" --no-deps
!pip install ".. /input/kerasapplications"
model_names = [
'roberta_base_noRawSenti',
'roberta_large',
'output_v10',
'output_v11',
]
for i, model_name in enumerate(model_names):
prob_start_tmp = np.load(f'start_{model_name}.npy')
 prob_end_tmp = np.load(f'end_{model_name}.npy')
 if i == 0:
prob_start = prob_start_tmp
prob_end = prob_end_tmp
else:
prob_start += prob_start_tmp
prob_end += prob_end_tmp
prob_start /= len(model_names)
 prob_end /= len(model_names)
 df_test['prob_start'] = prob_start.tolist()
df_test['prob_end'] = prob_end.tolist()<feature_engineering>
!pip install ".. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0"
def _get_pred_char(df, probs_start, probs_end):
df['start_idx'] = np.argmax(probs_start, axis=1)
 df['end_idx'] = probs_end.shape[1] - np.argmax(probs_end[:, ::-1], axis=1)- 1
df['prob_start'] = probs_start.tolist()
df['prob_end'] = probs_end.tolist()
idxs = np.where(df.start_idx > df.end_idx)
 for idx in idxs[0]:
prob_start = df.prob_start.values[idx]
prob_end = df.prob_end.values[idx]
start_idx = df.start_idx.values[idx]
end_idx = df.end_idx.values[idx]
if prob_start[start_idx] > prob_end[end_idx] or end_idx == 0:
end_idx = len(prob_start)- np.argmax(prob_end[start_idx:][::-1])- 1
else:
start_idx = np.argmax(prob_start[:end_idx])
 df['start_idx'].iloc[idx] = start_idx
df['end_idx'].iloc[idx] = end_idx
df['pred_char'] = df.apply(lambda x: x['text_clean'][x['start_idx']: x['end_idx']+1], axis=1)
 return df<feature_engineering>
gpus = tf.config.list_physical_devices('GPU')
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_virtual_device_configuration(
 gpu,
[tf.config.experimental.VirtualDeviceConfiguration(memory_limit=5600)])
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e )
def post_neutral(df):
df['select_pt'] = df.apply(lambda x: len(x['pred_char'].strip())/len(x['text_clean']), axis=1 ).values
raw_sents = ['neutral', 'sadness', 'worry', 'happiness', 'love', 'enthusiasm']
mm =(df['sentiment'] == 'neutral')&(df['raw_sentiment'].isin(raw_sents))
mm =(mm |\ 
(( df.select_pt > 0.85)&(df.sentiment.isin(['positive'])))|\ 
(( df.select_pt > 0.85)&(df.sentiment.isin(['negative'])))|\ 
(( df.select_pt < 0.2)&(df.sentiment.isin(['neutral']))))
df['pred_exp'] = df['pred_char'].values
df['pred_exp'].loc[mm] = df['text_clean'].loc[mm].values
print(f"
return df
def _post_shift_new(text, pred):
clean_text = " ".join(text.split())
 start_clean = clean_text.find(" ".join(pred.split()))
raw_pred = _convrt_back(text, pred, "neutral")
 raw_pred = raw_pred.strip()
start = text.find(raw_pred)
 end = start + len(raw_pred)
 extra_space = start - start_clean
if start>extra_space and extra_space>0:
if extra_space==1:
if text[start-1] in [',','.','?','!'] and text[start-2]!=' ':
start -= 1
elif extra_space==2:
start -= extra_space
if text[end-1] in [',','.','!','?','*']:
end -= 1
else:
end -=(extra_space-2)
 start -= extra_space
pred = text[start:end]
if pred.count("'")== 1:
if pred[0] == "'":
if text.find(pred)+ len(pred)< len(text)and text[text.find(pred)+ len(pred)] == "'":
pred += "'"
else:
if text.find(pred)- 1 >= 0 and text[text.find(pred)- 1] == "'":
pred = "'" + pred
return pred
def post_shift(df):
df['pred_final'] = df['pred_exp'].copy()
df['jac_text'] = df.apply(lambda x: jaccard(x['text'], x['pred_exp']), axis=1)
 mask =(df.sentiment != 'neutral')&(df.start_idx != 0)&(df.jac_text != 1)
 df['pred_final'].loc[mask] = df.apply(lambda x: _post_shift_new(x['text'], x['pred_exp']), axis=1 ).loc[mask].values
jac = df.apply(lambda x: jaccard(x['pred_exp'], x['pred_final']), axis=1)
 print(f"
return df<feature_engineering>
data_df_sample_submission = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv')
 test_files = os.listdir(".. /input/hpa-single-cell-image-classification/test")
 color_list = ["_red.png", "_green.png", "_yellow.png", "_blue.png"]
test_files_names = [re.sub(r'|'.join(map(re.escape, color_list)) , '', elem)for elem in test_files]
test_files_names = list(set(test_files_names))
d = []
for i, file in enumerate(test_files_names):
img = cv2.imread(f".. /input/hpa-single-cell-image-classification/test/{file}_red.png")
 height, width, channels = img.shape
d.append({
"ID" : file,
"ImageWidth" : width,
"ImageHeight": height,
"PredictionString" : "0 1 eNoLCAgIMAEABJkBdQ=="
})
 if i%50 == 0:
print(i)
 df_from_files = pd.DataFrame(d ).sort_values("ID" ).reset_index(drop=True)
 assert all(df_from_files == data_df_sample_submission), "Dataframes do not match"
POST_PROCESS=True
try:
df_test = _get_pred_char(df_test, np.array(df_test.prob_start.tolist()), np.array(df_test.prob_end.tolist()))
try:
if POST_PROCESS:
df_test = post_neutral(df_test)
 df_test = post_shift(df_test)
 df_test['selected_text'] = df_test['pred_final'].values
else:
df_test['selected_text'] = df_test['pred_char'].values
except:
df_test['selected_text'] = " "
except:
df_test = pd.merge(df_test[['textID']], df_pred1, how='left', on='textID' )<save_to_csv>
PATH_TEST = ".. /input/hpa-single-cell-image-classification/test"
MODEL_PATH = ".. /input/models-hpa/"
MODELS_LIST_EFFNET = [
"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_3.pth",
"efficientnet-b4_rgby_lr_0.0015_ADAM_focal1_g1.0_resize640_10pcTest_HEAVY_AUG_E3_F0.pth",
"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_E4_F4.pth",
]
MODELS_LIST_RESNEST = [
"resnest101_rgby_lr_0.002_SGD_polyoptim_focal1_g1.0_scheduler_largedataset_resize640_mediumaug_3.pth",
"resnest101_rgby_lr_0.002_SGD_focal1_g1.0_resize640_5pcTest_BS8_E3_F0.pth",
]
MODEL_LABELS_EFF = ".. /input/hpa-tensorflow-models/model_green.06-0.07.h5"
MODEL_LABELS_RN = ".. /input/hpa-tensorflow-models/model_rgb_resnext101.09-0.10.h5"
MODEL_LABELS_VIT = ".. /input/hpa-tensorflow-models/ggg_ViTB16_RedPlat_ADAMW_BCE_EPOCH12-VAL0.0957.h5"
df_test.to_csv("raw_prediction.csv", index=False)
 df_sub = df_test[['textID', 'selected_text']].copy()
df_sub.to_csv("submission.csv", index=False )<count_values>
class CFG:
debug=False
verbose = False
num_workers=8
model_name_effnet = 'efficientnet-b4'
model_name_resnest = 'resnest101'
size=640
seed=2002
classes = 19
color_mode = "rgby"
resnest = True
effnet = True
extra_model_for_labels = True
extra_model_is_tf = True
only_green_extra_model = True
color_mode_image_level = "rgb"
split = [0.6, 0.4, 0]
size_seg = None
split_image_level = [0.33, 0.33, 0.34, 0]
split_cam_level = [0.5, 0.5]
split_sigmoid_graboost = [0.5, 0.5]
sigmoid_factor = 2.0
sigmoid_move = 0.2
is_demo = len(data_df)==559
if CFG.is_demo:
data_df = data_df[:10]
batch_size_ = 4
print('Number of submissions above: ' + str(len(lbs)) )<import_modules>
Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, CoarseDropout 
)
 dataset_mean = [0.0994, 0.0466, 0.0606, 0.0879]
dataset_std = [0.1406, 0.0724, 0.1541, 0.1264]
def get_transforms(*, data_type):
if data_type == 'valid':
return Compose([
Resize(CFG.size, CFG.size),
ToTensorV2() ,
])
 elif data_type == 'test_green_model':
return Compose([
Resize(600, 600),
])
 elif data_type == 'test_green_model_torch':
return Compose([
Resize(CFG.size, CFG.size),
ToTensorV2() ,
] )
print('TF version',tf.__version__ )<define_variables>
def load_RGBY_image(image_id, path, mode="cam", image_size=None):
if mode == "green_model":
green = read_img_scale255(image_id, "green",path, image_size)
 stacked_images = np.transpose(np.array([green, green, green]),(1,2,0))
return stacked_images
if mode=="cam":
red = read_img(image_id, "red", path, image_size)
 green = read_img(image_id, "green",path, image_size)
 blue = read_img(image_id, "blue",path, image_size)
 yellow = read_img(image_id, "yellow",path, image_size)
 if CFG.color_mode == "rgby":
stacked_images = np.transpose(np.array([red, green, blue,yellow]),(1,2,0))
else:
stacked_images = np.transpose(np.array([red, green, blue]),(1,2,0))
return stacked_images
def read_img(image_id, color, path, image_size=None):
filename = f'{path}/{image_id}_{color}.png'
assert os.path.exists(filename), f'not found {filename}'
img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)
 if image_size is not None:
img = cv2.resize(img,(image_size, image_size))
if img.max() > 255:
img_max = img.max()
img =(img/255 ).astype('uint8')
 return img
def read_img_scale255(image_id, color, path, image_size=None):
filename = f'{path}/{image_id}_{color}.png'
assert os.path.exists(filename), f'not found {filename}'
img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)
 if image_size is not None:
img = cv2.resize(img,(image_size, image_size))
if img.max() > 255:
img_max = img.max()
img =(img/255 ).astype('uint8')/255
return img
def one_hot_embedding(label, classes):
vector = np.zeros(( classes), dtype = np.float32)
 if len(label)> 0:
vector[label] = 1.
 return vector
class HPADataset_Test(Dataset):
def __init__(self, ids, path=None, transforms=None, mode="cam"):
self.ids = ids
self.transforms = transforms
self.mode = mode
self.path = path
def __len__(self):
return len(self.ids)
 def __getitem__(self, idx):
_ids = self.ids.iloc[idx]
image = load_RGBY_image(_ids, self.path, self.mode)
 if self.transforms:
augmented = self.transforms(image=image)
 image = augmented['image']
return image, _ids
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
if CFG.color_mode_image_level == "ggg":
file_bytes = tf.io.read_file(path + "_green.png")
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
if CFG.color_mode_image_level == "rgb":
r = tf.io.read_file(path + "_red.png")
 g = tf.io.read_file(path + "_green.png")
 b = tf.io.read_file(path + "_blue.png")
 red = tf.io.decode_png(r, channels=1)
 blue = tf.io.decode_png(g, channels=1)
 green = tf.io.decode_png(b, channels=1)
 red = tf.image.resize(red, target_size)
 blue = tf.image.resize(blue, target_size)
 green = tf.image.resize(green, target_size)
 img = tf.stack([red, green, blue], axis=-1)
 img = tf.squeeze(img)
 img = tf.image.convert_image_dtype(img, tf.float32)/ 255
return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset_tf(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024, img_size=300,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None, target_size=(img_size, img_size))
if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset
test_paths = PATH_TEST + "/" + data_df['ID']
test_decoder_600 = build_decoder(with_labels=False, target_size=(600, 600))
test_decoder_384 = build_decoder(with_labels=False, target_size=(384, 384))
CFG.color_mode_image_level = "ggg"
dtest_tf_green_600 = build_dataset_tf(
 test_paths, bsize=batch_size_, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder_600 
)
 CFG.color_mode_image_level = "rgb"
dtest_tf_rgb_600 = build_dataset_tf(
 test_paths, bsize=batch_size_, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder_600 
)
 CFG.color_mode_image_level = "ggg"
dtest_tf_ggg_384 = build_dataset_tf(
 test_paths, bsize=batch_size_, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder_384
)
text = train['text'].values
selected_text = train['selected_text'].values.copy()
sentiments = train['sentiment'].values
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
ss = text[k].find(selected_text[k])
 if text[k][max(ss - 2, 0):ss] == ' ':
ss -= 2
if ss > 0 and text[k][ss - 1] == ' ':
ss -= 1
ee = ss + len(selected_text[k])
 if re.match(r' [^ ]', text[k])is not None:
ee -= 1
ss = max(0, ss)
 if ' ' in text[k][:ss] and sentiments[k] != 'neutral':
text1 = " ".join(text[k].split())
 sel = text1[ss:ee].strip()
if len(sel)> 1 and sel[-2] == ' ':
sel = sel[:-2]
selected_text[k] = sel
text1 = " "+" ".join(text[k].split())
 text2 = " ".join(selected_text[k].split() ).lstrip(".,;:")
 idx = text1.find(text2)
 if idx != -1:
chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
else:
chars = np.ones(( len(text1)))
 enc = tokenizer.encode(text1)
 offsets = enc.offsets
_toks = []
for i,(a,b)in enumerate(offsets):
sm = np.mean(chars[a:b])
 if(sm > 0.5 and chars[a] != 0):
_toks.append(i)
 toks = _toks
s_tok = sentiment_id[sentiments[k]]
input_ids[k, :len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1 
<feature_engineering>
class Yield_Images_Dataset(Dataset):
def __init__(self, csv_file, root=PATH_TEST, transform=None):
self.images_df = csv_file
self.transform = transform
self.root = root
def __len__(self):
return len(self.images_df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
_id = self.images_df["ID"].iloc[idx]
r = os.path.join(self.root, f'{_id}_red.png')
 y = os.path.join(self.root, f'{_id}_yellow.png')
 b = os.path.join(self.root, f'{_id}_blue.png')
 r = cv2.imread(r, 0)
 y = cv2.imread(y, 0)
 b = cv2.imread(b, 0)
 size = r.shape[0]
if CFG.size_seg == None:
ryb_image = np.stack(( r, y, b), axis=2)/255.
 blue_image = b/255.
 return blue_image, ryb_image, size, _id
if size != CFG.size_seg:
blue_image = cv2.resize(b,(CFG.size_seg, CFG.size_seg)) /255.
 ryb_image = np.stack(( r, y, b), axis=2)
 ryb_image = cv2.resize(ryb_image,(CFG.size_seg, CFG.size_seg)) /255.
 else:
ryb_image = np.stack(( r, y, b), axis=2)/255.
 blue_image = b/255.
 return blue_image, ryb_image, size, _id
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>

 NUCLEI_MODEL_URL, TWO_CHANNEL_CELL_MODEL_URL)
 NORMALIZE = {"mean": [124 / 255, 117 / 255, 104 / 255], "std": [1 /(0.0167 * 255)] * 3}
class CellSegmentator(object):

 def __init__(
 self,
nuclei_model="./nuclei_model.pth",
cell_model="./cell_model.pth",
model_width_height=None,
device="cuda",
multi_channel_model=True,
return_without_scale_restore=False,
scale_factor=0.25,
padding=False 
):
if device != "cuda" and device != "cpu" and "cuda" not in device:
raise ValueError(f"{device} is not a valid device(cuda/cpu)")
 if device != "cpu":
try:
assert torch.cuda.is_available()
except AssertionError:
print("No GPU found, using CPU.", file=sys.stderr)
 device = "cpu"
self.device = device
if isinstance(nuclei_model, str):
if not os.path.exists(nuclei_model):
print(
 f"Could not find {nuclei_model}.Downloading it now",
file=sys.stderr, 
)
 download_with_url(NUCLEI_MODEL_URL, nuclei_model)
 nuclei_model = torch.load(
 nuclei_model, map_location=torch.device(self.device)
)
 if isinstance(nuclei_model, torch.nn.DataParallel)and device == "cpu":
nuclei_model = nuclei_model.module
self.nuclei_model = nuclei_model.to(self.device)
 self.multi_channel_model = multi_channel_model
if isinstance(cell_model, str):
if not os.path.exists(cell_model):
print(
 f"Could not find {cell_model}.Downloading it now", file=sys.stderr 
)
 if self.multi_channel_model:
download_with_url(MULTI_CHANNEL_CELL_MODEL_URL, cell_model)
 else:
download_with_url(TWO_CHANNEL_CELL_MODEL_URL, cell_model)
 cell_model = torch.load(cell_model, map_location=torch.device(self.device))
self.cell_model = cell_model.to(self.device)
 self.model_width_height = model_width_height
self.return_without_scale_restore = return_without_scale_restore
self.scale_factor = scale_factor
self.padding = padding
def _image_conversion(self, images):
microtubule_imgs, er_imgs, nuclei_imgs = images
if self.multi_channel_model:
if not isinstance(er_imgs, list):
raise ValueError("Please speicify the image path(s)for er channels!")
 else:
if not er_imgs is None:
raise ValueError(
 "second channel should be None for two channel model predition!" 
)
 if not isinstance(microtubule_imgs, list):
raise ValueError("The microtubule images should be a list")
 if not isinstance(nuclei_imgs, list):
raise ValueError("The microtubule images should be a list")
 if er_imgs:
if not len(microtubule_imgs)== len(er_imgs)== len(nuclei_imgs):
raise ValueError("The lists of images needs to be the same length")
 else:
if not len(microtubule_imgs)== len(nuclei_imgs):
raise ValueError("The lists of images needs to be the same length")
 if not all(isinstance(item, np.ndarray)for item in microtubule_imgs):
microtubule_imgs = [
os.path.expanduser(item)for _, item in enumerate(microtubule_imgs)
 ]
nuclei_imgs = [
os.path.expanduser(item)for _, item in enumerate(nuclei_imgs)
 ]
microtubule_imgs = list(
 map(lambda item: imageio.imread(item), microtubule_imgs)
)
 nuclei_imgs = list(map(lambda item: imageio.imread(item), nuclei_imgs))
if er_imgs:
er_imgs = [os.path.expanduser(item)for _, item in enumerate(er_imgs)]
er_imgs = list(map(lambda item: imageio.imread(item), er_imgs))
if not er_imgs:
er_imgs = [
np.zeros(item.shape, dtype=item.dtype)
 for _, item in enumerate(microtubule_imgs)
 ]
cell_imgs = list(
 map(
 lambda item: np.dstack(( item[0], item[1], item[2])) ,
list(zip(microtubule_imgs, er_imgs, nuclei_imgs)) , 
)
)
 return cell_imgs
def _pad(self, image):
rows, cols = image.shape[:2]
self.scaled_shape = rows, cols
img_pad= cv2.copyMakeBorder(
 image,
32, 
(32 - rows % 32),
32, 
(32 - cols % 32),
cv2.BORDER_REFLECT, 
)
 return img_pad
def pred_nuclei(self, images):
def _preprocess(images):
if isinstance(images[0], str):
raise NotImplementedError('Currently the model requires images as numpy arrays, not paths.')
 self.target_shapes = [image.shape for image in images]
if self.model_width_height:
images = np.array([transform.resize(image,(self.model_width_height,self.model_width_height))
for image in images])
 else:
images = [transform.rescale(image, self.scale_factor)for image in images]
if self.padding:
images = [self._pad(image)for image in images]
nuc_images = np.array([np.dstack(( image[..., 2], image[..., 2], image[..., 2])) if len(image.shape)>= 3
else np.dstack(( image, image, image)) for image in images])
 nuc_images = nuc_images.transpose([0, 3, 1, 2])
 return nuc_images
def _segment_helper(imgs):
with torch.no_grad() :
mean = torch.as_tensor(NORMALIZE["mean"], device=self.device)
 std = torch.as_tensor(NORMALIZE["std"], device=self.device)
 imgs = torch.tensor(imgs ).float()
imgs = imgs.to(self.device)
 imgs = imgs.sub_(mean[:, None, None] ).div_(std[:, None, None])
 imgs = self.nuclei_model(imgs)
 imgs = F.softmax(imgs, dim=1)
 return imgs
preprocessed_imgs = _preprocess(images)
 predictions = _segment_helper(preprocessed_imgs)
 predictions = predictions.to("cpu" ).numpy()
predictions = [self._restore_scaling(util.img_as_ubyte(pred), target_shape)
 for pred, target_shape in zip(predictions, self.target_shapes)]
return predictions
def _restore_scaling(self, n_prediction, target_shape):

 n_prediction = n_prediction.transpose([1, 2, 0])
 if self.padding:
n_prediction = n_prediction[
32 : 32 + self.scaled_shape[0], 32 : 32 + self.scaled_shape[1],...
 ]
n_prediction[..., 0] = 0
if not self.return_without_scale_restore:
n_prediction = cv2.resize(
 n_prediction, 
(target_shape[0], target_shape[1]),
interpolation=cv2.INTER_AREA, 
)
 return n_prediction
def pred_cells(self, images, precombined=False):
def _preprocess(images):
self.target_shapes = [image.shape for image in images]
for image in images:
if not len(image.shape)== 3:
raise ValueError("image should has 3 channels")
 if self.model_width_height:
images = np.array([transform.resize(image,(self.model_width_height,self.model_width_height))
for image in images])
 else:
images = np.array([transform.rescale(image, self.scale_factor, multichannel=True)for image in images])
 if self.padding:
images = np.array([self._pad(image)for image in images])
 cell_images = images.transpose([0, 3, 1, 2])
 return cell_images
def _segment_helper(imgs):
with torch.no_grad() :
mean = torch.as_tensor(NORMALIZE["mean"], device=self.device)
 std = torch.as_tensor(NORMALIZE["std"], device=self.device)
 imgs = torch.tensor(imgs ).float()
imgs = imgs.to(self.device)
 imgs = imgs.sub_(mean[:, None, None] ).div_(std[:, None, None])
 imgs = self.cell_model(imgs)
 imgs = F.softmax(imgs, dim=1)
 return imgs
if not precombined:
images = self._image_conversion(images)
 preprocessed_imgs = _preprocess(images)
 predictions = _segment_helper(preprocessed_imgs)
 predictions = predictions.to("cpu" ).numpy()
predictions = [self._restore_scaling(util.img_as_ubyte(pred), target_shape)
 for pred, target_shape in zip(predictions, self.target_shapes)]
return predictions
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
IMAGE_SIZES = [1728, 2048, 3072, 4096]
predict_df_1728 = data_df[data_df.ImageWidth==IMAGE_SIZES[0]]
predict_df_2048 = data_df[data_df.ImageWidth==IMAGE_SIZES[1]]
predict_df_3072 = data_df[data_df.ImageWidth==IMAGE_SIZES[2]]
predict_df_4096 = data_df[data_df.ImageWidth==IMAGE_SIZES[3]]
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<normalization>
assert len(predict_df_1728)+ len(predict_df_2048)+ len(predict_df_3072)+ len(predict_df_4096)== len(data_df), "IMAGE SIZE DFS DONT MATCH SAMPLE SUBMISSION"
def modify_punc_length(text, selected_text):
m = re.search(r'[!\.\?]+$', selected_text)
 if m is None:
return selected_text
conti_punc = len(m.group())
 if conti_punc >= 4:
selected_text = selected_text[:-(conti_punc-2)]
elif conti_punc == 1:
tmp = re.sub(r"([\\\*\+\.\?\{\}\(\)\[\]\^\$\|])", r"\\\g<0>", selected_text)
 pat = re.sub(r" ", " +", tmp)
 m = re.search(pat, text)
 f_idx0 = m.start()
f_idx1 = m.end()
if f_idx1 != len(text)and text[f_idx1] in("!", ".", "?"):
f_idx1 += 1
selected_text = text[f_idx0:f_idx1]
return selected_text
def postprocess(row):
if row.original_text == '':
return row.normalized_text.strip()
original_text = row.original_text.replace('\t', '')
 y_start_char = row.y_start_char
y_end_char = row.y_end_char
y_selected_text = row.normalized_text[y_start_char:y_end_char].strip()
if(y_end_char < len(row.normalized_text)and row.sentiment != 'neutral' and
y_selected_text[-1] == '.' and 
(row.normalized_text[y_end_char] == '.' or
y_selected_text[-2] == '.')) :
y_selected_text = re.sub('\.+$', '.. ', y_selected_text)
 tmp = re.sub(r"([\\\*\+\.\?\{\}\(\)\[\]\^\$\|])", r"\\\g<0>", y_selected_text)
 pat = re.sub(r" ", " +", tmp)
 m = re.search(pat, original_text)
 if m is None:
print(row.normalized_text[y_start_char:y_end_char].strip())
 print(row.normalized_text)
 print(y_selected_text)
 return y_selected_text
ss2 = m.start()
ee2 = m.end()
if ' ' in original_text[:(ss2+ee2)//2]:
ss = y_start_char
ee = y_end_char
if ss > 1 and original_text[ss-1:ss+1] == '.. ' and original_text[ss+1] != '.' and row.sentiment != 'neutral':
ss -= 1
if row.sentiment == 'neutral':
ee -= 1
front_spaces = re.findall("^ +[^ ]", original_text[:ee2])
 if len(front_spaces)== 1:
ee += front_spaces[0].count(' ')
 elif len(front_spaces)> 1:
raise Excpetion("invalid process")
 for cnt_base in re.findall("[^ ] +[^ ]", original_text[:ee2].strip()):
ee += cnt_base[2:].count(' ')
 else:
ee += 1
st = original_text[ss:ee].lstrip(' ½¿')
 y_selected_text = re.sub(r'.$', '', st)
 else:
if(ee2 < len(original_text)-1 and original_text[ee2:ee2+2] in('.. ', '!!', '??', '(( ', ')) ')) :
ee2 += 1
if original_text[0] == ' ':
ss2 -= 1
y_selected_text = original_text[ss2:ee2].lstrip(' ½')
 if row.normalized_text[:y_end_char + 5] == " " + row.original_text[:ee2 + 4] and row.sentiment != 'neutral':
y_selected_text = modify_punc_length(original_text, y_selected_text)
 return y_selected_text<define_variables>
NUC_MODEL = ".. /input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth"
CELL_MODEL = ".. /input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth"
segmentator_even_faster = CellSegmentator(
 NUC_MODEL,
CELL_MODEL,
device="cuda",
multi_channel_model=True,
padding=True,
return_without_scale_restore=True
)
text = train['text'].values
selected_text = train['selected_text'].values.copy()
sentiments = train['sentiment'].values
ids = train['textID'].values
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,], attention_mask[idxV,], token_type_ids[idxV,]], verbose=DISPLAY, batch_size=128)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
text0 = text[k]
text1 = " " + " ".join(text[k].split())
 enc = tokenizer.encode(text1)
 aa = np.argmax(oof_start[k])
 if aa - 2 >= len(enc.offsets):
aa = 2
oof_end[k, :aa] = 0
bb = np.argmax(oof_end[k])
 try:
ss = enc.offsets[aa - 2][0]
ee = enc.offsets[bb - 2][1]
st = text1[ss:ee].strip()
except IndexError:
ss = 0
ee = len(text1)
 st = text1
row = pd.Series(dict(
 original_text=text0,
normalized_text=text1,
sentiment=sentiments[k],
y_start_char=ss,
y_end_char=ee, 
))
try:
st = postprocess(row)
 except:
print(k)
 st = tokenizer.decode(enc.ids[aa-2:bb-1])
 sc = jaccard(st,selected_text[k])
 all.append(sc)
 jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
yield_ims_1728 = Yield_Images_Dataset(predict_df_1728)
 yield_ims_2048 = Yield_Images_Dataset(predict_df_2048)
 yield_ims_3072 = Yield_Images_Dataset(predict_df_3072)
 yield_ims_4096 = Yield_Images_Dataset(predict_df_4096)
 dataloader_ims_seg_1728 = DataLoader(yield_ims_1728, batch_size=24,
shuffle=False, num_workers=0)
 dataloader_ims_seg_2048 = DataLoader(yield_ims_2048, batch_size=12,
shuffle=False, num_workers=0)
 dataloader_ims_seg_3072 = DataLoader(yield_ims_3072, batch_size=3,
shuffle=False, num_workers=0)
 dataloader_ims_seg_4096 = DataLoader(yield_ims_4096, batch_size=3,
shuffle=False, num_workers=0)
 dataloaders_all_sizes = [dataloader_ims_seg_1728, dataloader_ims_seg_2048, dataloader_ims_seg_3072, dataloader_ims_seg_4096]
start_time = time.time()
even_faster_outputs = []
output_ids = []
batch_size = 24
sizes_list = []
im_proc = 0
for i, dataloader_ims_seg in enumerate(dataloaders_all_sizes):
print(f"GETTING IMAGE SIZES: {IMAGE_SIZES[i]}, BATCHES: {len(dataloader_ims_seg)}")
 print
for blue_images, ryb_images, sizes, _ids in dataloader_ims_seg:
print(f"SEGMENT COUNT: {im_proc}")
 blue_batch = blue_images.numpy()
ryb_batch = ryb_images.numpy()
nuc_segmentations = segmentator_even_faster.pred_nuclei(blue_batch)
 cell_segmentations = segmentator_even_faster.pred_cells(ryb_batch, precombined=True)
 for data_id, nuc_seg, cell_seg, size in zip(_ids, nuc_segmentations, cell_segmentations, sizes):
_, cell = utils.label_cell(nuc_seg, cell_seg)
 even_faster_outputs.append(np.ubyte(cell))
output_ids.append(data_id)
 sizes_list.append(size.numpy())
 im_proc += len(_ids)
 del dataloader_ims_seg
print(time.time() - start_time)
 cell_masks_df = pd.DataFrame(list(zip(output_ids, even_faster_outputs,sizes_list)) ,
columns=["ID", "mask", "ori_size"] )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<categorify>
cell_masks_df = cell_masks_df.set_index('ID')
 cell_masks_df = cell_masks_df.reindex(index=data_df['ID'])
 cell_masks_df = cell_masks_df.reset_index()
all = []
for k in range(input_ids_t.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 aa = np.argmax(preds_start[k])
 if aa - 2 >= len(enc.offsets):
aa = 2
preds_end[k, :aa] = -1
bb = np.argmax(preds_end[k])
 try:
ss = enc.offsets[aa - 2][0]
ee = enc.offsets[bb - 2][1]
st = text1[ss:ee].strip()
except IndexError:
ss = 0
ee = len(text1)
 st = text1
row = pd.Series(dict(
 original_text=test.loc[k,'text'],
normalized_text=text1,
sentiment=test.loc[k,'sentiment'],
y_start_char=ss,
y_end_char=ee, 
))
try:
st = postprocess(row)
 except:
print(k)
 st = tokenizer.decode(enc.ids[aa-2:bb-1])
 all.append(st )<save_to_csv>
del sizes_list
del even_faster_outputs
del output_ids
del segmentator_even_faster
del yield_ims_1728
del yield_ims_2048
del yield_ims_3072
del yield_ims_4096
del dataloader_ims_seg_1728
del dataloader_ims_seg_2048
del dataloader_ims_seg_3072
del dataloader_ims_seg_4096
del dataloaders_all_sizes
libc = ctypes.CDLL("libc.so.6")
 libc.malloc_trim(0)
 gc.collect()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<set_options>
libc = ctypes.CDLL("libc.so.6")
 libc.malloc_trim(0)
 gc.collect()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
%matplotlib inline
pd.set_option('max_colwidth', 40 )<define_variables>
X_test = data_df["ID"]
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<feature_engineering>
assert list(X_test)== list(cell_masks_df["ID"]), "X_Test and cellmask dont match"
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<init_hyperparams>
test_data = HPADataset_Test(X_test, path=PATH_TEST, transforms=get_transforms(data_type='valid'), mode="cam")
 test_data_green_model = HPADataset_Test(X_test, path=PATH_TEST,transforms=get_transforms(data_type='valid'), mode="green_model" )
Dropout_new = 0.15
n_split = 5
lr = 3e-5<train_model>
def swish(x, beta=1.0):
return x * torch.sigmoid(beta*x)
 def get_all_cams(batch_cam_scaled, model, scales, ims_per_batch):
bs = ims_per_batch
with torch.no_grad() :
ori_w, ori_h = CFG.size, CFG.size
strided_up_size =(CFG.size, CFG.size)
 all_scale_cams = torch.from_numpy(np.zeros(( bs, len(scales), 19, CFG.size, CFG.size)) ).cuda()
all_scale_preds = torch.from_numpy(np.zeros(( bs, len(scales), 19)) ).cuda()
num_channels = 4
for i, images in enumerate(batch_cam_scaled):

 with torch.cuda.amp.autocast() :
logits, features = model(images, with_cam=True)
 features = swish(features)
 logits = logits.reshape(bs*4//bs//4, 4, bs, 19 ).mean(1 ).view(bs*4//4, 19)
 all_scale_preds[:, i, :] = logits
features = torch.cat([features[0:bs], features[bs:bs*2].flip(2), features[bs*2:bs*3].flip(3), torch.flip(features[bs*3:bs*4],(3,2)) ])
 size_feats = features.shape[-1]
features = features.reshape(bs*4//bs//4, 4, bs, 19, size_feats, size_feats ).sum(1 ).view(bs*4//4, 19, size_feats, size_feats)
 cams = F.interpolate(features,(CFG.size, CFG.size), mode='bicubic', align_corners=False)
 all_scale_cams[:, i, :, :, :] = cams
all_logits = np.sum(all_scale_preds.detach().cpu().numpy() , axis=1)
 all_cams = np.sum(all_scale_cams.detach().cpu().numpy() , axis=1)
 print("CAMS DONE")
 return {"hr_cams": all_cams, "logits" : all_logits}
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
label_names = [
'0-Nucleoplasm',
'1-Nuclear membrane',
'2-Nucleoli',
'3-Nucleoli fibrillar center',
'4-Nuclear speckles',
'5-Nuclear bodies',
'6-Endoplasmic reticulum',
'7-Golgi apparatus',
'8-Intermediate filaments',
'9-Actin filaments',
'10-Microtubules',
'11-Mitotic spindle',
'12-Centrosome',
'13-Plasma membrane',
'14-Mitochondria',
'15-Aggresome',
'16-Cytosol',
'17-Vesicles + cytosolic patterns',
'18-Negative'
]
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
def encode_binary_mask(mask: np.ndarray)-> t.Text:

 if mask.dtype != np.bool:
raise ValueError(
 "encode_binary_mask expects a binary mask, received dtype == %s" %
mask.dtype)
 mask = np.squeeze(mask)
 if len(mask.shape)!= 2:
raise ValueError(
 "encode_binary_mask expects a 2d mask, received shape == %s" %
mask.shape)
 mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)
 mask_to_encode = mask_to_encode.astype(np.uint8)
 mask_to_encode = np.asfortranarray(mask_to_encode)
 encoded_mask = coco_mask.encode(mask_to_encode)[0]["counts"]
binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)
 base64_str = base64.b64encode(binary_str)
 return base64_str
def get_all_encoded_cells(mask):
print(mask.shape)
 cell_masks = []
for i in range(1, np.max(mask)+1):
enc_mask = encode_binary_mask(( mask == i))
cell_masks.append(enc_mask)
 return cell_masks
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<define_variables>
def resize_mask(mask):
resized_mask = resize_full_mask(mask, CFG.size)
 cell_masks = []
for i in range(1, np.max(mask)+1):
cell_masks.append(( resized_mask == i))
return cell_masks
def resize_full_mask(mask, size):
resized_mask = cv2.resize(mask,(size,size),interpolation=cv2.INTER_NEAREST_EXACT)
 return resized_mask
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<compute_test_metric>
def get_pred_string(mask_probas, cell_masks_fullsize_enc):
assert len(mask_probas)== len(cell_masks_fullsize_enc), "Probas have different length than masks"
string = ""
for enc_mask, mask_proba in zip(cell_masks_fullsize_enc, mask_probas):
for cls, proba in enumerate(mask_proba):
string += str(cls)+ " " + str(proba)+ " " + enc_mask.decode("utf-8")+ " "
return string
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )<string_transform>
PATH_SCALER_GRADBOOST = ".. /input/scaler-and-gradboost"
scaler_resnest0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_resnest0.pkl", 'rb'))
scaler_resnest1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_resnest1.pkl", 'rb'))
scaler_effnet0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_effnet0.pkl", 'rb'))
scaler_effnet1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_effnet1.pkl", 'rb'))
scaler_effnet2 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_effnet2.pkl", 'rb'))
model_gradboost_resnest0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_resnest0.pkl", 'rb'))
model_gradboost_resnest1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_resnest1.pkl", 'rb'))
model_gradboost_effnet0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet0.pkl", 'rb'))
model_gradboost_effnet1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet1.pkl", 'rb'))
model_gradboost_effnet2 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet2.pkl", 'rb'))
def get_prob_from_cams_masks(cams, masks, labels, labels_from_labelmodel, verbose=True, typ=None):
print(f"GETTING MODEL {typ}")
 if typ=="resnest0":
scaler = scaler_resnest0
model_gradboost = model_gradboost_resnest0
if typ=="resnest1":
scaler = scaler_resnest1
model_gradboost = model_gradboost_resnest1
if typ=="resnest2":
scaler = scaler_resnest1
model_gradboost = model_gradboost_resnest1
if typ=="effnet0":
scaler = scaler_effnet0
model_gradboost = model_gradboost_effnet0
if typ=="effnet1":
scaler = scaler_effnet1
model_gradboost = model_gradboost_effnet1
if typ=="effnet2":
scaler = scaler_effnet2
model_gradboost = model_gradboost_effnet2
if typ=="effnet3":
scaler = scaler_effnet2
model_gradboost = model_gradboost_effnet2
masks_probas = np.zeros(( len(masks), 19))
for i, mask in enumerate(masks):
for label, cam in enumerate(cams):
cam_by_mask = np.multiply(mask, cam)
 cam_mask_product = np.multiply(cam_by_mask, labels[label])
 masks_probas[i, label] = np.sum(cam_mask_product)
 if verbose:
print(f"MASK: {i} PROB-RAW: {masks_probas[i, :]}")
 print("--------------------------------------")
 std_scaler = preprocessing.RobustScaler().fit(masks_probas.reshape(-1, 1))
for i, mask in enumerate(masks):
std_scaled = std_scaler.transform(masks_probas[i, :].reshape(-1, 1)) [:,0]
model_scaled = scaler.transform(masks_probas[i, :].reshape(-1, 1)) [:,0]
if verbose:
print(f"MASK: {i} STD-Scaled: {std_scaled}")
 print(f"MASK: {i} Model-Scaled: {model_scaled}")
 print("--------------------------------------")
 sigmoid_probas = sigmoid_factor(std_scaled, factor=CFG.sigmoid_factor, move=CFG.sigmoid_move)
 gradboost_probas = model_gradboost.predict(model_scaled.reshape(-1, 1))
if verbose:
print(f"MASK: {i} PROB-SIGMOID: {sigmoid_probas}")
 print(f"MASK: {i} PROB-GradBoost: {gradboost_probas}")
 print("--------------------------------------")
 if CFG.extra_model_for_labels:
masks_probas[i, :] = sigmoid_probas*CFG.split_sigmoid_graboost[0] + gradboost_probas*CFG.split_sigmoid_graboost[1]
masks_probas[i, :] = CFG.split[0] * masks_probas[i, :] + CFG.split[1] * labels_from_labelmodel + CFG.split[2] * labels
if verbose:
print(f"MASK: {i} PROB-WITH-LABELMODEL: {masks_probas[i, :]}")
 return masks_probas
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
class Classifier_EffNet(nn.Module, ABC_Model):
def __init__(self, backbone, num_classes=19):
super(Classifier_EffNet, self ).__init__()
self.enet = EfficientNet.from_name(backbone, num_classes=num_classes, in_channels=3, include_top=False)
 dict_sizes = {
'efficientnet-b0' : 1280,
'efficientnet-b1' : 1280,
'efficientnet-b2' : 1408,
'efficientnet-b3' : 1536,
'efficientnet-b4' : 1792,
'efficientnet-b5' : 2048
}
size_conv2d = dict_sizes[backbone]
self.classifier = nn.Conv2d(size_conv2d, num_classes, 1, bias=False)
 self.num_classes = num_classes
self.initialize([self.classifier])
 def forward(self, x, with_cam=False):
x = self.enet.extract_features(x)
 if with_cam:
features = self.classifier(x)
 logits = self.global_average_pooling_2d(features)
 return logits, features
else:
x = self.global_average_pooling_2d(x, keepdims=True)
 logits = self.classifier(x ).view(-1, self.num_classes)
 return logits
class Classifier_EffNet_GREEN(nn.Module, ABC_Model):
def __init__(self, backbone, num_classes=19):
super(Classifier_EffNet_GREEN, self ).__init__()
self.enet = EfficientNet.from_name(backbone, num_classes=num_classes, in_channels=3, include_top=False)
 dict_sizes = {
'efficientnet-b0' : 1280,
'efficientnet-b1' : 1280,
'efficientnet-b2' : 1408,
'efficientnet-b3' : 1536,
'efficientnet-b4' : 1792,
'efficientnet-b5' : 2048,
'efficientnet-b7' : 2560,
}
self.dense = nn.Linear(dict_sizes[backbone],19)
 self.initialize([self.dense])
 def forward(self, x, with_cam=False):
x = self.enet.extract_features(x)
 x = self.global_average_pooling_2d(x)
 logits = self.dense(x)
 return logits
class FixedBatchNorm(nn.BatchNorm2d):
def forward(self, x):
return F.batch_norm(x, self.running_mean, self.running_var, self.weight, self.bias, training=False, eps=self.eps)
 def group_norm(features):
return nn.GroupNorm(4, features)
 class Backbone(nn.Module, ABC_Model):
def __init__(self, model_name, num_classes=20, mode='fix', segmentation=False):
super().__init__()
self.mode = mode
if self.mode == 'fix':
self.norm_fn = FixedBatchNorm
else:
self.norm_fn = nn.BatchNorm2d
if 'resnet' in model_name:
self.model = resnet.ResNet(resnet.Bottleneck, resnet.layers_dic[model_name], strides=(2, 2, 2, 1), batch_norm_fn=self.norm_fn)
 state_dict = model_zoo.load_url(resnet.urls_dic[model_name])
 state_dict.pop('fc.weight')
 state_dict.pop('fc.bias')
 self.model.load_state_dict(state_dict)
 else:
if segmentation:
dilation, dilated = 4, True
else:
dilation, dilated = 2, False
self.model = eval("resnest." + model_name )(pretrained=False, dilated=dilated, dilation=dilation, norm_layer=self.norm_fn)
 del self.model.avgpool
del self.model.fc
self.stage1 = nn.Sequential(self.model.conv1,
self.model.bn1,
self.model.relu,
self.model.maxpool)
 self.stage2 = nn.Sequential(self.model.layer1)
 self.stage3 = nn.Sequential(self.model.layer2)
 self.stage4 = nn.Sequential(self.model.layer3)
 self.stage5 = nn.Sequential(self.model.layer4)
 class Classifier(Backbone):
def __init__(self, model_name, num_classes=20, mode='fix'):
super().__init__(model_name, num_classes, mode)
 self.classifier = nn.Conv2d(2048, num_classes, 1, bias=False)
 self.num_classes = num_classes
self.initialize([self.classifier])
 def forward(self, x, with_cam=False):
x = self.stage1(x)
 x = self.stage2(x)
 x = self.stage3(x)
 x = self.stage4(x)
 x = self.stage5(x)
 if with_cam:
features = self.classifier(x)
 logits = self.global_average_pooling_2d(features)
 return logits, features
else:
x = self.global_average_pooling_2d(x, keepdims=True)
 logits = self.classifier(x ).view(-1, self.num_classes)
 return logits
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )<define_variables>
def get_separate_labels(ims, model, model_for_labels_state, ims_per_batch, verbose=False):
bs = ims_per_batch
model.load_state_dict(model_for_labels_state["model_state_dict"])
 with torch.no_grad() :
image_batch = copy.deepcopy(ims)
 image_batch = image_batch.float()
image_batch_augs_fl2 = image_batch.flip(2)
 image_batch_augs_fl3 = image_batch.flip(3)
 image_batch_augs_fl32 = torch.flip(image_batch,(3,2))
images = torch.cat([image_batch, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)
 images = images.cuda()
print(images.shape)
 with torch.cuda.amp.autocast() :
logits = model(images, with_cam=True)
 logits = logits.reshape(bs*4//bs//4, 4, bs, 19 ).mean(1 ).view(bs*4//4, 19)
 labels = torch.sigmoid(logits)
 if verbose:
print("LABELS FROM LABELS MODEL TORCH")
 print(labels)
 return labels.cpu().numpy()
def get_separate_labels_tf(ims, model, verbose=False, name=None):
bs = ims_per_batch
image_batch = copy.deepcopy(ims)
 image_batch_fl_lr = tf.image.flip_left_right(image_batch)
 image_batch_fl_up = tf.image.flip_up_down(image_batch)
 image_batch_fl_up_lr = tf.image.flip_up_down(image_batch_fl_lr)
 ims = tf.concat([image_batch, image_batch_fl_lr, image_batch_fl_up, image_batch_fl_up_lr], axis=0)
 labels = model.predict(ims, verbose=1)
 labels = labels.reshape(bs*4//bs//4, 4, bs, 19 ).mean(1)
 if verbose:
print(f"LABELS FROM LABELS MODEL {name}")
 print(labels)
 return labels[0]
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )<compute_train_metric>
sys.path.append(".. /input/faustomorales-vitkeras/vit_keras/")
 CONFIG_B = {
"dropout": 0.1,
"mlp_dim": 3072,
"num_heads": 12,
"num_layers": 12,
"hidden_size": 768,
}
class TransformerBlock(tf.keras.layers.Layer):

 def __init__(self, *args, num_heads=12, mlp_dim=3072, dropout=0.1, **kwargs):
super().__init__(*args, **kwargs)
 self.num_heads = num_heads
self.mlp_dim = mlp_dim
self.dropout = dropout
def build(self, input_shape):
self.att = MultiHeadSelfAttention(
 num_heads=self.num_heads,
name="MultiHeadDotProductAttention_1", 
)
 self.mlpblock = tf.keras.Sequential(
 [
tf.keras.layers.Dense(
 self.mlp_dim,
activation="linear",
name=f"{self.name}/Dense_0", 
),
tf.keras.layers.Lambda(
 lambda x: tf.keras.activations.gelu(x, approximate=False)
)
 if hasattr(tf.keras.activations, "gelu")
 else tf.keras.layers.Lambda(
 lambda x: tfa.activations.gelu(x, approximate=False)
),
tf.keras.layers.Dropout(self.dropout),
tf.keras.layers.Dense(input_shape[-1], name=f"{self.name}/Dense_1"),
tf.keras.layers.Dropout(self.dropout),
],
name="MlpBlock_3", 
)
 self.layernorm1 = tf.keras.layers.LayerNormalization(
 epsilon=1e-6, name="LayerNorm_0" 
)
 self.layernorm2 = tf.keras.layers.LayerNormalization(
 epsilon=1e-6, name="LayerNorm_2" 
)
 self.dropout = tf.keras.layers.Dropout(self.dropout)
 def call(self, inputs, training):
x = self.layernorm1(inputs)
 x, weights = self.att(x)
 x = self.dropout(x, training=training)
 x = x + inputs
y = self.layernorm2(x)
 y = self.mlpblock(y)
 return x + y, weights
class MultiHeadSelfAttention(tf.keras.layers.Layer):
def __init__(self, *args, num_heads=12, **kwargs):
super().__init__(*args, **kwargs)
 self.num_heads = num_heads
def build(self, input_shape):
hidden_size = input_shape[-1]
num_heads = self.num_heads
if hidden_size % num_heads != 0:
raise ValueError(
 f"embedding dimension = {hidden_size} should be divisible by number of heads = {num_heads}" 
)
 self.hidden_size = hidden_size
self.projection_dim = hidden_size // num_heads
self.query_dense = tf.keras.layers.Dense(hidden_size, name="query")
 self.key_dense = tf.keras.layers.Dense(hidden_size, name="key")
 self.value_dense = tf.keras.layers.Dense(hidden_size, name="value")
 self.combine_heads = tf.keras.layers.Dense(hidden_size, name="out")
 def attention(self, query, key, value):
score = tf.matmul(query, key, transpose_b=True)
 dim_key = tf.cast(tf.shape(key)[-1], score.dtype)
 scaled_score = score / tf.math.sqrt(dim_key)
 weights = tf.nn.softmax(scaled_score, axis=-1)
 output = tf.matmul(weights, value)
 return output, weights
def separate_heads(self, x, batch_size):
x = tf.reshape(x,(batch_size, -1, self.num_heads, self.projection_dim))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, inputs):
batch_size = tf.shape(inputs)[0]
query = self.query_dense(inputs)
 key = self.key_dense(inputs)
 value = self.value_dense(inputs)
 query = self.separate_heads(query, batch_size)
 key = self.separate_heads(key, batch_size)
 value = self.separate_heads(value, batch_size)
 attention, weights = self.attention(query, key, value)
 attention = tf.transpose(attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(attention,(batch_size, -1, self.hidden_size))
output = self.combine_heads(concat_attention)
 return output, weights
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )<feature_engineering>
if CFG.resnest:
model_resnest = Classifier(CFG.model_name_resnest, CFG.classes, mode="normal")
 if CFG.color_mode == "rgby":
weight = model_resnest.model.conv1[0].weight.clone()
model_resnest.model.conv1[0] = nn.Conv2d(4, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
 with torch.no_grad() :
model_resnest.model.conv1[0].weight[:, :3] = weight
model_resnest.model.conv1[0].weight[:, 3] = model_resnest.model.conv1[0].weight[:, 0]
model_resnest.to(device)
 model_resnest.eval()
model_states_resnet = [torch.load(MODEL_PATH + f"{model}")for model in MODELS_LIST_RESNEST]
if CFG.effnet:
model_effnet = Classifier_EffNet(CFG.model_name_effnet)
 if CFG.color_mode == "rgby":
model_effnet.enet._conv_stem.in_channels = 4
model_effnet.enet._conv_stem.weight = torch.nn.Parameter(torch.cat([model_effnet.enet._conv_stem.weight, model_effnet.enet._conv_stem.weight[:, 0:1, :, :]], axis=1))
model_effnet.to(device)
 model_effnet.eval()
model_states_effnet = [torch.load(MODEL_PATH + f"{model}")for model in MODELS_LIST_EFFNET]
if CFG.extra_model_for_labels:
if CFG.extra_model_is_tf:
model_green_eff = tf.keras.models.load_model(MODEL_LABELS_EFF)
 model_green_rn = tf.keras.models.load_model(MODEL_LABELS_RN)
 model_green_vit = tf.keras.models.load_model(MODEL_LABELS_VIT, custom_objects={
'ClassToken': layers.ClassToken,
'AddPositionEmbs' : layers.AddPositionEmbs,
'TransformerBlock' : TransformerBlock,
'MultiHeadSelfAttention' : MultiHeadSelfAttention})
 else:
model_for_labels_state = torch.load(MODEL_LABELS)
 model_green = Classifier_EffNet_GREEN("efficientnet-b7")
 model_green.to(device)
 model_green.eval()
dl_test = DataLoader(test_data, batch_size=batch_size_, shuffle=False, num_workers=0)
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']<feature_engineering>
def inference_one_batch(batch_cam, batch_ids_cam, batch_seg, ims_per_batch, batch_eff_tf=None, batch_rn_tf_600=None, batch_vit_tf_384=None, show_image=True, show_seg=True, verbose=True):
batch_ids_seg = tuple(batch_seg["ID"])
 print(batch_ids_cam)
 print(batch_ids_seg)
 assert batch_ids_cam == batch_ids_seg, "IDS OF SEGMENTATION AND CAMS DONT MATCH"
print(f"GETTING {ims_per_batch} CELL MASKS")
 cell_mask_list = batch_seg["mask"]
cell_mask_sizes = batch_seg["ori_size"]
cell_masks_full_size = []
for e,(mask, size)in enumerate(zip(cell_mask_list, cell_mask_sizes)) :
cell_masks_full_size.append(resize_full_mask(mask,size))
res_cell_masks = [resize_mask(cell_mask)for cell_mask in cell_mask_list]
if show_seg:
show_seg_cells(batch_cam, res_cell_masks, batch_ids_seg, batch_cam)
 del cell_mask_list
cell_masks_fullsize_enc_list = []
for cell_mask in cell_masks_full_size:
cell_masks_fullsize_enc = get_all_encoded_cells(cell_mask)
 cell_masks_fullsize_enc_list.append(cell_masks_fullsize_enc)
 del cell_masks_full_size
print(f"ENCODED {len(cell_masks_fullsize_enc_list)} CELL MASKS")
 labels_model_eff = get_separate_labels_tf(batch_eff_tf, model_green_eff, verbose=verbose, name=MODEL_LABELS_EFF)
 labels_model_rn = get_separate_labels_tf(batch_rn_tf_600, model_green_rn, verbose=verbose, name=MODEL_LABELS_RN)
 labels_model_vit = get_separate_labels_tf(batch_vit_tf_384, model_green_vit, verbose=verbose, name=MODEL_LABELS_VIT)
 labels_model = labels_model_eff*CFG.split_image_level[0] + labels_model_rn*CFG.split_image_level[1] + labels_model_vit*CFG.split_image_level[2]
print("LABELS FROM LABEL MODEL")
 print(labels_model)
 print("RESIZE IMAGES")
 batch_cam_scaled = []
scales = [1.0, 1.3, 1.6]
st = time.time()
for i, scale in enumerate(scales):
image_batch_pil = torch.from_numpy(np.zeros(( ims_per_batch, 4, round(CFG.size*scale), round(CFG.size*scale))))
image_batch = copy.deepcopy(batch_cam)
 for j, im in enumerate(image_batch):
im = ToPILImage()(im)
 im = im.resize(( round(CFG.size*scale), round(CFG.size*scale)) , resample=PIL.Image.BICUBIC)
 im = torchvision.transforms.functional.to_tensor(im)*255
image_batch_pil[j] = im
image_batch_resized = image_batch_pil.float()
image_batch_augs_fl2 = image_batch_resized.flip(2)
 image_batch_augs_fl3 = image_batch_resized.flip(3)
 image_batch_augs_fl32 = torch.flip(image_batch_resized,(3,2))
images = torch.cat([image_batch_resized, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)
 images = images.cuda()
batch_cam_scaled.append(images)
 print(f"TIME FOR RESIZING WITH PIL {time.time() - st}")
 if CFG.resnest:
print("GETTING CAMS AND PREDS RESNEST")
 mask_probas_resnest_folds = []
folds_resnest = len(MODELS_LIST_RESNEST)
 all_ims_resnest = []
time_spent_mask_probas = []
for f, model_state_rn in enumerate(model_states_resnet):
all_hr_cams, sig_labels = get_hrcams_vis(batch_cam_scaled, show_image, model_resnest, model_state_rn, scales, ims_per_batch)
 mask_probas_resnest_batches = []
print(f"GETTING MASKS PROBAS FOLD {f}")
 for b,(cams, mask, sig_label, label_model)in enumerate(zip(all_hr_cams, res_cell_masks, sig_labels, labels_model)) :
print(f"GETTING MASKS PROBAS BATCH {b}")
 mask_probas_resnest = get_prob_from_cams_masks(cams, mask, sig_label, label_model, verbose, typ=f"resnest{f}")
 if verbose:
print(mask_probas_resnest)
 mask_probas_resnest_batches.append(mask_probas_resnest)
 mask_probas_resnest_folds.append(mask_probas_resnest_batches)
 for b in range(ims_per_batch):
all_ims_resnest.append(np.mean(np.array([mask_probas_resnest_folds[i][b] for i in range(folds_resnest)]), axis=0))
if CFG.effnet:
print("GETTING CAMS AND PREDS EFFICIENTNET")
 mask_probas_effnet_folds = []
folds_effnet = len(MODELS_LIST_EFFNET)
 all_ims_effnet = []
time_spent_mask_probas = []
for f, model_state_eff in enumerate(model_states_effnet):
all_hr_cams, sig_labels = get_hrcams_vis(batch_cam_scaled, show_image, model_effnet, model_state_eff, scales, ims_per_batch)
 mask_probas_effnet_batches = []
print(f"GETTING MASKS PROBAS FOLD {f}")
 for b,(cams, mask, sig_label, label_model)in enumerate(zip(all_hr_cams, res_cell_masks, sig_labels, labels_model)) :
print(f"GETTING MASKS PROBAS BATCH {b}")
 s_t = time.time()
mask_probas_effnet = get_prob_from_cams_masks(cams, mask, sig_label, label_model, verbose, typ=f"effnet{f}")
 if verbose:
print(mask_probas_effnet)
 mask_probas_effnet_batches.append(mask_probas_effnet)
 mask_probas_effnet_folds.append(mask_probas_effnet_batches)
 for b in range(ims_per_batch):
all_ims_effnet.append(np.mean(np.array([mask_probas_effnet_folds[i][b] for i in range(folds_effnet)]), axis=0))
if CFG.resnest and CFG.effnet:
mask_probas = CFG.split_cam_level[0]*np.array(all_ims_effnet)+ CFG.split_cam_level[1]*np.array(all_ims_resnest)
 elif CFG.resnest and not CFG.effnet:
mask_probas = all_ims_resnest
elif CFG.effnet and not CFG.resnest:
mask_probas = all_ims_effnet
if verbose:
print(mask_probas)
 return batch_ids_cam, sizes, cell_masks_fullsize_enc_list, mask_probas
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1<string_transform>
libc = ctypes.CDLL("libc.so.6")
 libc.malloc_trim(0)
 gc.collect()
gc.collect()
gc.collect()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3<feature_engineering>
if CFG.is_demo:
index = 0
batch0, ids0 = next(itertools.islice(dl_test, index, None))
if CFG.extra_model_is_tf:
batch_tf_green = next(itertools.islice(dtest_tf_green_600, index, None))
batch_tf_rgb_600 = next(itertools.islice(dtest_tf_rgb_600, index, None))
batch_tf_rgb_384 = next(itertools.islice(dtest_tf_ggg_384, index, None))
print(batch_tf_green.shape)
 print(batch_tf_rgb_600.shape)
 print(batch_tf_rgb_384.shape)
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )<filter>
df = pd.DataFrame(columns=["image_id", "pred"])
 i = 0
start_time = time.time()
ims_done = 0
for i,(( batch_cam, batch_ids_cam),(batch_tf_green_600),(batch_tf_rgb_600),(batch_tf_ggg_384)) in enumerate(zip(dl_test, dtest_tf_green_600, dtest_tf_rgb_600, dtest_tf_ggg_384)) :
ims_per_batch = len(batch_ids_cam)
 batch_seg = cell_masks_df[i*batch_size_ : i*batch_size_ + batch_size_]
ids, sizes, cell_masks_fullsize_enc_list, probas = inference_one_batch(batch_cam, batch_ids_cam, batch_seg, ims_per_batch, batch_eff_tf=batch_tf_green_600, batch_rn_tf_600=batch_tf_rgb_600, batch_vit_tf_384=batch_tf_ggg_384, show_image=False, show_seg=False, verbose=False)
 for id, proba, cell_mask_enc in zip(ids, probas, cell_masks_fullsize_enc_list):
pred_string = get_pred_string(proba, cell_mask_enc)
 d = {"image_id":id, "pred": pred_string}
df = df.append(d, ignore_index=True)
 ims_done += ims_per_batch
print(f"---{ims_done} IMAGES DONE---")
 torch.cuda.empty_cache()
gc.collect()
print("--- %s seconds ---" %(time.time() - start_time))
train_outlier = train[train['res'] == 0].reset_index(drop=True)
 train_outlier<define_variables>
sub = pd.merge(
 data_df,
df,
how="left",
left_on='ID',
right_on='image_id', 
)
 def isNaN(num):
return num != num
for i, row in sub.iterrows() :
if isNaN(row['pred']): continue
sub.PredictionString.loc[i] = row['pred']
<predict_on_test><EOS> 
if all(df_from_files == data_df_sample_submission):
sub.to_csv("submission.csv",index=False)