Kaggle competition: Human Protein Atlas - Single Cell Classification
<SOS> metric: OpenImagesObjDetectionSegmentationAP Kaggle data source: human-protein-atlas-single-cell-classification<train_model>
!pip install -q ".. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl"
position = 'End'
train_data = get_training_data(train_df_se, position)
 model_path = get_model_out_path(position)
 train(train_data, model_path, n_iter=3, cont=False )<predict_on_test>
import pandas as pd
import numpy as np
import os
import tqdm
import cv2
import matplotlib.pyplot as plt
import seaborn as sns
import gc
import base64
from pycocotools import _mask as coco_mask
import typing as t
import zlib
from torch.utils.data import Dataset, DataLoader
import albumentations as A
from albumentations.pytorch.transforms import ToTensorV2
import torch
import torch.nn as nn
from collections import OrderedDict
import ttach as tta
def predict_entities(text, model):
doc = model(text)
 ent_array = []
for ent in doc.ents:
start = text.find(ent.text)
 end = start + len(ent.text)
 new_int = [start, end, ent.label_]
if new_int not in ent_array:
ent_array.append([start, end, ent.label_])
 if len(ent_array)> 0:
selected_text = text[ent_array[0][0]: ent_array[0][1]]
else:
selected_text = text
return selected_text<load_pretrained>
df = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv' )
selected_texts = []
MODELS_BASE_PATH = '.. /working/models/'
if MODELS_BASE_PATH is not None:
print("Loading Models from ", MODELS_BASE_PATH)
 model_Start = spacy.load(MODELS_BASE_PATH + 'model_Start')
 model_End = spacy.load(MODELS_BASE_PATH + 'model_End')
 for index, row in validation_df_se.iterrows() :
text = row.text_se
output_str = ""
if row.position == 'Start-Neutral' or row.position == 'End-Neutral':
selected_texts.append(text)
 elif row.position == 'Start':
selected_texts.append(predict_entities(text, model_Start))
elif row.position == 'End':
selected_texts.append(predict_entities(text, model_End))
validation_df_se['ner_predicted_text'] = selected_texts<define_variables>
ROOT = '.. /input/hpa-single-cell-image-classification/'
train_or_test = 'test'
counter = 0
for index, row in validation_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
counter += 1
print("Different word count in validation set =",counter )<define_variables>
def get_cropped_cell(img, msk):
bmask = msk.astype(int)[...,None]
masked_img = img * bmask
true_points = np.argwhere(bmask)
 top_left = true_points.min(axis=0)
 bottom_right = true_points.max(axis=0)
 cropped_arr = masked_img[top_left[0]:bottom_right[0]+1,top_left[1]:bottom_right[1]+1]
return cropped_arr
counter = 0
for index, row in validation_df_se.iterrows() :
if row["text_se"] == row["selected_text_se"]:
counter+=1
print("Same word count in validation set =",counter )<define_variables>
def get_stats(cropped_cell):
x =(cropped_cell/255.0 ).reshape(-1,3 ).mean(0)
 x2 =(( cropped_cell/255.0)**2 ).reshape(-1,3 ).mean(0)
 return x, x2
counter = 0
for index, row in validation_df_se.iterrows() :
if row["ner_predicted_text"] != row["text_se"]:
counter+=1
print("Changed word by NER in validation set =",counter )<define_variables>
def read_img(image_id, color, train_or_test='test', image_size=None):
filename = f'{ROOT}/{train_or_test}/{image_id}_{color}.png'
assert os.path.exists(filename), f'not found {filename}'
img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)
 if image_size is not None:
img = cv2.resize(img,(image_size, image_size))
if img.max() > 255:
img_max = img.max()
img =(img/255 ).astype('uint8')
 return img
counter = 0
for index, row in validation_df_se.iterrows() :
if row["ner_predicted_text"] != row["text_se"]:
if row["ner_predicted_text"] != row["selected_text_se"]:
counter+=1
print("Changed word by NER in validation set but unsuccesfull changing=",counter )<predict_on_test>
def encode_binary_mask(mask: np.ndarray)-> t.Text:

 if mask.dtype != np.bool:
raise ValueError(
 "encode_binary_mask expects a binary mask, received dtype == %s" %
mask.dtype)
 mask = np.squeeze(mask)
 if len(mask.shape)!= 2:
raise ValueError(
 "encode_binary_mask expects a 2d mask, received shape == %s" %
mask.shape)
 mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)
 mask_to_encode = mask_to_encode.astype(np.uint8)
 mask_to_encode = np.asfortranarray(mask_to_encode)
 encoded_mask = coco_mask.encode(mask_to_encode)[0]["counts"]
binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)
 base64_str = base64.b64encode(binary_str)
 return base64_str.decode('ascii' )
predict_entities("badly??!?!??!!?", model_End )<filter>
warnings.filterwarnings("ignore", category=SourceChangeWarning)
 NUC_MODEL = ".. /input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth"
CELL_MODEL = ".. /input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth"
segmentator = cellsegmentator.CellSegmentator(
 NUC_MODEL,
CELL_MODEL,
scale_factor=0.25,
device="cuda",
padding=True,
multi_channel_model=True, 
);
for index, row in validation_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
print(row["text_se"], row["selected_text_se"] )<feature_engineering>
df = pd.read_csv('./cell_df.csv' )
for index, row in validation_df.iterrows() :
predicted_words = row["predicted_text"].split()
predicted_words[0] = validation_df_se["ner_predicted_text"][2*index]
predicted_words[-1] = validation_df_se["ner_predicted_text"][(2*index)+1]
validation_df["predicted_text"][index] = " ".join(predicted_words )<compute_test_metric>
valid_transforms = A.Compose([
A.Resize(width=224, height=224),
A.Normalize() ,
ToTensorV2() ,
])
 class CellDataset(Dataset):
def __init__(self, data_dir, csv_file, transform=None):
super().__init__()
self.data_dir = data_dir
self.df = csv_file
self.transforms = transform
self.img_ids = self.df['image_id'].values
self.cell_ids = self.df['cell_id'].values
def __len__(self):
return len(self.img_ids)
 def get_image(self, index):
image_id = self.img_ids[index]
cell_id = self.cell_ids[index]
img_path = os.path.join(self.data_dir, 'cells', image_id + '_' + str(cell_id)+ '.jpg')
 img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = self.transforms(image=img)
 img = img['image']
return img
def __getitem__(self, index):
x = self.get_image(index)
 return x
totalJaccard = 0
for index, row in validation_df.iterrows() :
predict = row.predicted_text
true = row.selected_text
j = jaccard(predict, true)
 totalJaccard += j
meanJaccard = totalJaccard / len(validation_df['predicted_text'])
 print("NER Validation results : ", meanJaccard )<load_pretrained>
test_dataset = CellDataset(data_dir='', csv_file=df, transform=valid_transforms )
selected_texts = []
MODELS_BASE_PATH = '.. /working/models/'
if MODELS_BASE_PATH is not None:
print("Loading Models from ", MODELS_BASE_PATH)
 model_Start = spacy.load(MODELS_BASE_PATH + 'model_Start')
 model_End = spacy.load(MODELS_BASE_PATH + 'model_End')
 for index, row in test_df_se.iterrows() :
text = row.text_se
output_str = ""
if row.position == 'Start-Neutral' or row.position == 'End-Neutral':
selected_texts.append(text)
 elif row.position == 'Start':
selected_texts.append(predict_entities(text, model_Start))
elif row.position == 'End':
selected_texts.append(predict_entities(text, model_End))
test_df_se['ner_predicted_text'] = selected_texts<feature_engineering>
test_loader = DataLoader(test_dataset,
batch_size=8,
shuffle=False,
num_workers=4,
drop_last=False )
for index, row in test_df.iterrows() :
predicted_words = row["selected_text"].split()
predicted_words[0] = test_df_se["ner_predicted_text"][2*index]
predicted_words[-1] = test_df_se["ner_predicted_text"][(2*index)+1]
test_df["selected_text"][index] = " ".join(predicted_words )<save_to_csv>
class Net(nn.Module):
def __init__(self, name = 'efficientnet_b0', num_classes=19):
super(Net, self ).__init__()
self.model = timm.create_model(name, pretrained=False, num_classes=num_classes)
 def forward(self, x):
out = self.model(x)
 return out
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )<set_options>
def update_state_dict(state_dict):
new_state_dict = OrderedDict()
for key in state_dict.keys() :
new_state_dict['.'.join(key.split('.')[1:])] = state_dict[key]
return new_state_dict
warnings.filterwarnings("ignore")
 warnings.filterwarnings("ignore",category=UserWarning )<define_variables>
model_b1_f0 = Net(name = 'efficientnet_b1')
 model_b1_f0.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b1-224-fold-0/epoch1-valid_loss_epoch0.118.pth')))
 model_b1_f0.cuda() ;
model_b1_f0.eval() ;
model_b1_f1 = Net(name = 'efficientnet_b1')
 model_b1_f1.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b1-224-fold-1/epoch1-valid_loss_epoch0.117.pth')))
 model_b1_f1.cuda() ;
model_b1_f1.eval() ;
model_b1_f2 = Net(name = 'efficientnet_b1')
 model_b1_f2.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b1-224-fold-4/epoch1-valid_loss_epoch0.119.pth')))
 model_b1_f2.cuda() ;
model_b1_f2.eval() ;
model_b1_f3 = Net(name = 'efficientnet_b1')
 model_b1_f3.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b1-224-fold-3/epoch1-valid_loss_epoch0.116.pth')))
 model_b1_f3.cuda() ;
model_b1_f3.eval() ;
model_b1_f4 = Net(name = 'efficientnet_b1')
 model_b1_f4.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b1-224-fold-44/epoch1-valid_loss_epoch0.117.pth')))
 model_b1_f4.cuda() ;
model_b1_f4.eval() ;
model_b0_f0 = Net(name = 'efficientnet_b0')
 model_b0_f0.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b0-224-fold-0/epoch1-valid_loss_epoch0.119.pth')))
 model_b0_f0.cuda() ;
model_b0_f0.eval() ;
model_b0_f1 = Net(name = 'efficientnet_b0')
 model_b0_f1.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b0-224-fold-1/epoch1-valid_loss_epoch0.118.pth')))
 model_b0_f1.cuda() ;
model_b0_f1.eval() ;
model_b0_f2 = Net(name = 'efficientnet_b0')
 model_b0_f2.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b0-224-fold-2/epoch1-valid_loss_epoch0.120.pth')))
 model_b0_f2.cuda() ;
model_b0_f2.eval() ;
model_b0_f3 = Net(name = 'efficientnet_b0')
 model_b0_f3.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b0-224-fold-3/epoch1-valid_loss_epoch0.117.pth')))
 model_b0_f3.cuda() ;
model_b0_f3.eval() ;
model_b0_f4 = Net(name = 'efficientnet_b0')
 model_b0_f4.load_state_dict(update_state_dict(torch.load('.. /input/efficientnet-b0-224-fold-4/epoch8-valid_loss_epoch0.117.pth')))
 model_b0_f4.cuda() ;
model_b0_f4.eval() ;
copyfile(src = ".. /input/utils-v10/utilsv10.py", dst = ".. /working/utilsv10.py")
 copyfile(src = ".. /input/utils-v10/dataset10.py", dst = ".. /working/dataset10.py")
 copyfile(src = ".. /input/utils-v10/dataset11.py", dst = ".. /working/dataset11.py")
 load_model, save_model, set_seed, write_event, evaluate, get_predicts_from_token_logits)
<load_from_csv>
pred = torch.FloatTensor()
pred = pred.cuda()
df_pred = pd.read_csv('.. /input/tweet-sentiment-fast/test_all_post_finetune_0608.csv')
 df_pred1 = pd.read_csv('.. /input/tweet-sentiment-fast/test_all_post_finetune_large.csv')
 df_train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 df_test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 df_test.loc[:, 'selected_text'] = df_test.text.values
df_test['text_clean'] = df_test['text'].apply(lambda x: " ".join(x.split()))
df_full = pd.read_csv('/kaggle/input/complete-tweet-sentiment-extraction-data/tweet_dataset.csv')
 df_full = df_full[df_full.text.notnull() ].copy()
df_full['text_clean'] = df_full['text'].apply(lambda x: " ".join(x.split()))
df_full = df_full.drop_duplicates(subset='text')
 df_full = df_full[~df_full.aux_id.isin(df_train.textID)]
df_full.rename(columns={'sentiment': 'raw_sentiment'}, inplace=True )<string_transform>
tta1 = tta.ClassificationTTAWrapper(model_b1_f0, tta.aliases.flip_transform())
 tta2 = tta.ClassificationTTAWrapper(model_b1_f1, tta.aliases.flip_transform())
 tta3 = tta.ClassificationTTAWrapper(model_b1_f2, tta.aliases.flip_transform())
 tta4 = tta.ClassificationTTAWrapper(model_b1_f3, tta.aliases.flip_transform())
 tta5 = tta.ClassificationTTAWrapper(model_b1_f4, tta.aliases.flip_transform())
 tta6 = tta.ClassificationTTAWrapper(model_b0_f0, tta.aliases.flip_transform())
 tta7 = tta.ClassificationTTAWrapper(model_b0_f1, tta.aliases.flip_transform())
 tta8 = tta.ClassificationTTAWrapper(model_b0_f2, tta.aliases.flip_transform())
 tta9 = tta.ClassificationTTAWrapper(model_b0_f3, tta.aliases.flip_transform())
 tta10 = tta.ClassificationTTAWrapper(model_b0_f4, tta.aliases.flip_transform() )
def find_sentiment(textID, text, sentiment):
text_clean = " ".join(text.split())
 if textID in df_full.aux_id.values:
return df_full['raw_sentiment'].loc[df_full.aux_id==textID].values[0]
elif text in df_full.text.values:
return df_full['raw_sentiment'].loc[df_full.text==text].values[0]
elif text_clean in df_full.text_clean.values:
return df_full['raw_sentiment'].loc[df_full.text_clean==text_clean].values[0]
else:
return sentiment<feature_engineering>
with torch.no_grad() :
for inp in tqdm.tqdm(test_loader):
bs, c, h, w = inp.size()
input_var = torch.autograd.Variable(inp.view(-1, c, h, w ).cuda())
 output =(tta1(input_var)+tta2(input_var)+\
tta3(input_var)+tta4(input_var)+\
tta5(input_var)+tta6(input_var)+\
tta7(input_var)+tta8(input_var)+tta9(input_var)+tta10(input_var)) /10
output_mean = output.view(bs, -1)
 pred = torch.cat(( pred, output_mean.data), 0 )
df_test['raw_sentiment'] = df_test.apply(lambda x: find_sentiment(x['textID'], x['text'], x['sentiment']), axis=1)
 na_mask = df_test.raw_sentiment.isnull()
print(na_mask.sum() )<load_from_csv>
def isNaN(num):
return num != num
test = df_test.copy()
tokenizer = AutoTokenizer.from_pretrained('.. /input/roberta-base/', do_lower_case=False)
 class Args:
post = True
tokenizer = tokenizer
offset = 4
batch_size = 4
workers = 0
args = Args()
class Args11:
post = True
tokenizer = tokenizer
offset = 7
batch_size = 4
workers = 0
args11 = Args11()<load_pretrained>
cell_df = pd.read_csv('./cell_df.csv')
 cell_df['cls'] = ''
class TweetModel(nn.Module):
def __init__(self, pretrain_path=None, dropout=0.2, config=None):
super(TweetModel, self ).__init__()
if config is not None:
self.bert = AutoModel.from_config(config)
 else:
config = AutoConfig.from_pretrained(pretrain_path, output_hidden_states=True)
 self.bert = AutoModel.from_pretrained(
 pretrain_path, cache_dir=None, config=config)
 self.cnn = nn.Conv1d(self.bert.config.hidden_size*3, self.bert.config.hidden_size, 3, padding=1)
 self.gelu = nn.GELU()
self.whole_head = nn.Sequential(OrderedDict([ 
('dropout', nn.Dropout(0.1)) , 
('l1', nn.Linear(self.bert.config.hidden_size*3, 256)) , 
('act1', nn.GELU()), 
('dropout', nn.Dropout(0.1)) , 
('l2', nn.Linear(256, 2))
]))
self.se_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.inst_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.dropout = nn.Dropout(0.1)
 def forward(self, inputs, masks, token_type_ids=None, input_emb=None):
_, pooled_output, hs = self.bert(
 inputs, masks, token_type_ids=token_type_ids, inputs_embeds=input_emb)
 seq_output = torch.cat([hs[-1],hs[-2],hs[-3]], dim=-1)
 avg_output = torch.sum(seq_output*masks.unsqueeze(-1), dim=1, keepdim=False)
 avg_output = avg_output/torch.sum(masks, dim=-1, keepdim=True)
 whole_out = self.whole_head(avg_output)
 seq_output = self.gelu(self.cnn(seq_output.permute(0,2,1)).permute(0,2,1))
se_out = self.se_head(self.dropout(seq_output))
inst_out = self.inst_head(self.dropout(seq_output))
return whole_out, se_out[:, :, 0], se_out[:, :, 1], inst_out<find_best_model_class>
threshold = 0.0
for i in range(pred_torch.shape[0]):
p = torch.nonzero(pred_torch[i] > threshold ).squeeze().numpy().tolist()
if type(p)!= list: p = [p]
if len(p)== 0: cls = [(pred_torch[i].argmax().item() , pred_torch[i].max().item())]
else: cls = [(x, pred_torch[i][x].item())for x in p]
cell_df['cls'].loc[i] = cls
def predict_wu(model: nn.Module, valid_df, valid_loader, args, progress=False)-> Dict[str, float]:
model.eval()
all_end_pred, all_whole_pred, all_start_pred, all_inst_out = [], [], [], []
if progress:
tq = tqdm.tqdm(total=len(valid_df))
with torch.no_grad() :
for tokens, types, masks, _, _, _, _, _, _, _ in valid_loader:
if progress:
batch_size = tokens.size(0)
 tq.update(batch_size)
 masks = masks.cuda()
tokens = tokens.cuda()
types = types.cuda()
whole_out, start_out, end_out, inst_out = model(tokens, masks, types)
 all_whole_pred.append(torch.softmax(whole_out, dim=-1)[:,1].cpu().numpy())
 inst_out = torch.softmax(inst_out, dim=-1)
 for idx in range(len(start_out)) :
length = torch.sum(masks[idx,:] ).item() -1
all_start_pred.append(torch.softmax(start_out[idx, args.offset:length], axis=-1 ).cpu())
 all_end_pred.append(torch.softmax(end_out[idx, args.offset:length], axis=-1 ).cpu())
 all_inst_out.append(inst_out[idx,:,1].cpu())
 assert all_start_pred[-1].dim() ==1
all_whole_pred = np.concatenate(all_whole_pred)
 if progress:
tq.close()
return all_whole_pred, all_start_pred, all_end_pred, all_inst_out<statistical_test>
def combine(r):
cls = r[0]
enc = r[1]
classes = [str(c[0])+ ' ' + str(c[1])+ ' ' + enc for c in cls]
return ' '.join(classes)
 combine(cell_df[['cls', 'enc']].loc[24]);
def _convrt_prob_partI(text, prob, max_char_len=160):
clean_text = " ".join(text.split())
 new_prob = []
p1, p2 = 0, 0
while p1 < len(text):
if text[p1] not in [" ", "\t", "\xa0"]:
if text[p1] == clean_text[p2]:
new_prob.append(prob[p1])
 p1 += 1
p2 += 1
else:
if p1 + 1 < len(text)and text[p1+1] not in [" ", "\t", "\xa0"]:
if clean_text[p2] == " ":
new_prob.append(prob[p1])
 p1 += 1
p2 += 1
else:
p1 += 1
else:
p1 += 1
if len(new_prob)< max_char_len:
new_prob = new_prob +(max_char_len - len(new_prob)) *[0]
return new_prob<find_best_model_class>
cell_df['pred'] = cell_df[['cls', 'enc']].apply(combine, axis=1)
 cell_df.head()
def get_prediction_partI(weights_path, test, test_loader, args, output_name):
config = RobertaConfig.from_pretrained('.. /input/roberta-base', output_hidden_states=True)
 model = TweetModel(config=config)
 all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = [], [], [], []
for fold in range(10):
load_model(model, f'{weights_path}/best-model-%d.pt' % fold)
 model.cuda()
fold_whole_preds, fold_start_preds, fold_end_preds, fold_inst_preds = predict_wu(model, test, test_loader, args, progress=True)
 all_whole_preds.append(fold_whole_preds)
 all_start_preds.append(fold_start_preds)
 all_end_preds.append(fold_end_preds)
 all_inst_preds.append(fold_inst_preds)
 all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = ensemble(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test)
 word_preds, inst_word_preds, scores = get_predicts_from_token_logits(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test, args)
 start_char_prob, end_char_prob = get_char_prob(all_start_preds, all_end_preds, test, args)
 test['start_char_prob'] = start_char_prob
test['end_char_prob'] = end_char_prob
test['selected_text'] = word_preds
test['prob_start'] = test.apply(lambda x: _convrt_prob_partI(x['text'], x['start_char_prob']), axis=1)
 test['prob_end'] = test.apply(lambda x: _convrt_prob_partI(x['text'], x['end_char_prob']), axis=1)
 test.to_pickle(f'{output_name}.pkl')
 np.save(f"start_{output_name}.npy", np.array(test['prob_start'].tolist()))
np.save(f"end_{output_name}.npy", np.array(test['prob_end'].tolist()))<create_dataframe>
subm = cell_df.groupby(['image_id'])['pred'].apply(lambda x: ' '.join(x)).reset_index()
subm.head()
collator = MyCollator()
test_set = TrainDataset(test, None, tokenizer=tokenizer, mode='test', offset=args.offset)
 test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, collate_fn=collator,
num_workers=args.workers)
 get_prediction_partI(weights_path='.. /input/roberta-v10-10',
test=test,
test_loader=test_loader,
args=args,
output_name='output_v10', 
)
 mem = psutil.virtual_memory()
print(f'{mem.percent:5} - {mem.free/1024**3:10.2f} - {mem.available/1024**3:10.2f} - {mem.used/1024**3:10.2f}' )<load_pretrained>
sample_submission = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv')
 sample_submission.head()
collator = MyCollator()
test_set = TrainDataset11(test, None, tokenizer=tokenizer, mode='test', offset=args11.offset)
 test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, collate_fn=collator,
num_workers=args.workers)
 get_prediction_partI(weights_path='.. /input/roberta-v11-10',
test=test,
test_loader=test_loader,
args=args11,
output_name='output_v11', 
)
 mem = psutil.virtual_memory()
print(f'{mem.percent:5} - {mem.free/1024**3:10.2f} - {mem.available/1024**3:10.2f} - {mem.used/1024**3:10.2f}' )<define_variables>
sub = pd.merge(
 sample_submission,
subm,
how="left",
left_on='ID',
right_on='image_id', 
)
 sub.head()
copyfile(src = ".. /input/tweet-sentiment/common.py", dst = ".. /working/common.py")
 copyfile(src = ".. /input/tweet-sentiment/dataset.py", dst = ".. /working/dataset.py")
 copyfile(src = ".. /input/tweet-sentiment/models.py", dst = ".. /working/models.py")
 copyfile(src = ".. /input/tweet-sentiment/metrics.py", dst = ".. /working/metrics.py")
 copyfile(src = ".. /input/tweet-sentiment/utils.py", dst = ".. /working/utils.py")
 copyfile(src = ".. /input/tweet-sentiment/predict_fn.py", dst = ".. /working/predict_fn.py")
 copyfile(src = ".. /input/tweet-sentiment/nlp_albumentations.py", dst = ".. /working/nlp_albumentations.py")
 copyfile(src = ".. /input/tweet-sentiment/transform.py", dst = ".. /working/transform.py")
 copyfile(src = ".. /input/tweet-sentiment/run_inference_kaggle.py", dst = ".. /working/run_inference_kaggle.py")
 set_seed(42)
<define_variables>
def isNaN(num):
return num != num
for i, row in sub.iterrows() :
if isNaN(row['pred']): continue
sub.PredictionString.loc[i] = row['pred']
def get_model(model_name):
if model_name.startswith('roberta_base'):
model_info = {
'name': 'roberta-base',
'model_path': '.. /input/roberta-base' if ON_KAGGLE else 'roberta-base',
'from_pretrained': False if ON_KAGGLE else True,
'vocab_file': '.. /input/roberta-base/vocab.json',
'merges_file': '.. /input/roberta-base/merges.txt',
}
elif model_name.startswith('roberta_large'):
model_info = {
'name': 'roberta-large',
'model_path': '.. /input/roberta-large' if ON_KAGGLE else 'roberta-large',
'from_pretrained': False if ON_KAGGLE else True,
'vocab_file': '.. /input/roberta-large/vocab.json',
'merges_file': '.. /input/roberta-large/merges.txt',
}
else:
raise RuntimeError('%s is not implemented.' % model_name)
 model = TweetModel_v2(model_info)
 tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=model_info['vocab_file'],
merges_file=model_info['merges_file'],
lowercase=False,
add_prefix_space=True 
)
 return model, tokenizer<define_variables>
sub.to_csv('submission.csv', index=False )
N_FOLD = 10
POST_PROCESS = True
params = {
'models': [
'roberta_base_noRawSenti',
'roberta_large',
],
'batch_size': 4,
'workers': 1 if ON_KAGGLE else 8,
'max_len': 192,
'folds': list(x for x in range(N_FOLD)) ,
'limit': 0,
}
path_lib = {
'roberta_base': '.. /input/roberta-base/',
'roberta_base_noRawSenti': '.. /input/roberta-base-bs32-v2-0608',
'roberta_large': '.. /input/roberta-large-bs32-v2-0608',
}<predict_on_test>
sub.to_csv('submission.csv', index=False )
<create_dataframe><EOS> 
!rm cell_df.csv