Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<feature_engineering>
!pip install.. /input/apex01cp37cp37mlinux-x86-64whl/apex-0.1-cp37-cp37m-linux_x86_64.whl
!pip install --upgrade.. /input/tokenizers0011cp37/tokenizers-0.0.11-cp37-cp37m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl
encode_dict = {}
path = '.. /input/train_simplified/'
filenames = glob.glob(os.path.join(path, '*.csv'))
filenames = sorted(filenames)
 print(len(filenames))
for ix, filename in enumerate(filenames):
class_name = filename.split('/')[-1].split('.')[0].replace(' ', '_')
 encode_dict[class_name] = ix
decode_dict = {value:key for key, value in encode_dict.items() }<load_from_csv>
from transformers import BertConfig, BertTokenizer, RobertaConfig, RobertaTokenizer
class ClassDataset(Dataset):
def __init__(self, csv_file_path, mode='train', nrows=15000, skiprows=0, size=128):
super().__init__()
self.df = pd.read_csv(csv_file_path, usecols=['drawing'], nrows=nrows, skiprows=0)
 self.mode = mode
self.size = size
if self.mode == 'train':
self.class_name = csv_file_path.split('/')[-1].split('.')[0].replace(' ', '_')
 self.label = encode_dict[self.class_name]
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
raw_strokes = ast.literal_eval(self.df.drawing[index])
 image = draw_cv2(raw_strokes, size=self.size)
 image = image.transpose(2, 0, 1)
 if self.mode == 'train':
return(image/255 ).astype('float32'), self.label
else:
return(image/255 ).astype('float32')
<create_dataframe>
def bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len):
class TFQADataset(Dataset):
def __init__(self, id_list):
self.id_list=id_list
def __len__(self):
return len(self.id_list)
 def __getitem__(self, index):
return self.id_list[index]
class Collator(object):
def __init__(self, data_dict, new_token_dict, tokenizer, max_seq_len=384, max_question_len=64):
self.data_dict = data_dict
self.new_token_dict = new_token_dict
self.tokenizer = tokenizer
self.max_seq_len = max_seq_len
self.max_question_len = max_question_len
def _get_input_ids(self, doc_id, candidate_index):
data = self.data_dict[doc_id]
question_tokens = self.tokenizer.tokenize(data['question_text'])[:self.max_question_len]
doc_words = data['document_text'].split()
max_answer_tokens = self.max_seq_len-len(question_tokens)-3
candidate = data['long_answer_candidates'][candidate_index]
candidate_start = candidate['start_token']
candidate_end = candidate['end_token']
candidate_words = doc_words[candidate_start:candidate_end]
for i, word in enumerate(candidate_words):
if re.match(r'<.+>', word):
if word in self.new_token_dict:
candidate_words[i] = self.new_token_dict[word]
else:
candidate_words[i] = '<'
words_to_tokens_index = []
candidate_tokens = []
for i, word in enumerate(candidate_words):
words_to_tokens_index.append(len(candidate_tokens))
tokens = self.tokenizer.tokenize(word)
 if len(candidate_tokens)+len(tokens)> max_answer_tokens:
break
for token in tokens:
candidate_tokens.append(token)
 input_tokens = ['[CLS]'] + question_tokens + ['[SEP]'] + candidate_tokens + ['[SEP]']
input_ids = self.tokenizer.convert_tokens_to_ids(input_tokens)
 return input_ids, words_to_tokens_index, len(input_ids), len(question_tokens)+2
def __call__(self, batch_ids):
batch_size = len(batch_ids)
 batch_input_ids_temp = []
batch_seq_len = []
batch_offset = []
batch_words_to_tokens_index = []
for i,(doc_id, candidate_index)in enumerate(batch_ids):
input_ids, words_to_tokens_index, seq_len, offset = self._get_input_ids(doc_id, candidate_index)
 batch_input_ids_temp.append(input_ids)
 batch_seq_len.append(seq_len)
 batch_offset.append(offset)
 batch_words_to_tokens_index.append(words_to_tokens_index)
 batch_max_seq_len = max(batch_seq_len)
 batch_input_ids = np.zeros(( batch_size, batch_max_seq_len), dtype=np.int64)
 batch_token_type_ids = np.ones(( batch_size, batch_max_seq_len), dtype=np.int64)
 for i in range(batch_size):
input_ids = batch_input_ids_temp[i]
batch_input_ids[i, :len(input_ids)] = input_ids
batch_token_type_ids[i, :len(input_ids)] = [0 if k<=input_ids.index(102)else 1 for k in range(len(input_ids)) ]
batch_attention_mask = batch_input_ids > 0
return torch.from_numpy(batch_input_ids), torch.from_numpy(batch_attention_mask), torch.from_numpy(batch_token_type_ids), batch_words_to_tokens_index, batch_offset, batch_max_seq_len
class BertForQuestionAnswering(BertPreTrainedModel):
def __init__(self, config):
super(BertForQuestionAnswering, self ).__init__(config)
 self.bert = BertModel(config)
 self.qa_outputs = nn.Linear(config.hidden_size, 2)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.init_weights()
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 sequence_output = outputs[0]
pooled_output = outputs[1]
qa_logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = qa_logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 pooled_output = self.dropout(pooled_output)
 classifier_logits = self.classifier(pooled_output)
 return start_logits, end_logits, classifier_logits
max_seq_len = 360
max_question_len = 64
batch_size = int(384/4)
 model_path = '.. /models/models/bert/'
config = BertConfig.from_pretrained(model_path)
 config.num_labels = 5
config.vocab_size = 30531
tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = BertForQuestionAnswering.from_pretrained(model_dir, config=config)
 new_token_dict = {
'<P>':'qw1',
'<Table>':'qw2',
'<Tr>':'qw3',
'<Ul>':'qw4',
'<Ol>':'qw5',
'<Fl>':'qw6',
'<Li>':'qw7',
'<Dd>':'qw8',
'<Dt>':'qw9',
}
new_token_list = [
'qw1',
'qw2',
'qw3',
'qw4',
'qw5',
'qw6',
'qw7',
'qw8',
'qw9',
]
num_added_toks = tokenizer.add_tokens(new_token_list)
 model.resize_token_embeddings(len(tokenizer))
model.cuda()
optimizer = optim.Adam(model.parameters() , lr=1e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)
 if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
 test_datagen = TFQADataset(id_list=id_candidate_list_sorted)
 test_collate = Collator(data_dict=data_dict,
new_token_dict=new_token_dict,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
max_question_len=max_question_len)
 test_generator = DataLoader(dataset=test_datagen,
collate_fn=test_collate,
batch_size=batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
 model.eval()
list_offset = []
list_words_to_tokens_index = []
test_prob1 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob2 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob3 = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 for j,(batch_input_ids, batch_attention_mask, batch_token_type_ids, batch_words_to_tokens_index, batch_offset, batch_max_seq_len)in tqdm(enumerate(test_generator)) :
with torch.no_grad() :
start = j*batch_size
end = start+batch_size
if j == len(test_generator)-1:
end = len(test_generator.dataset)
 batch_input_ids = batch_input_ids.cuda()
batch_attention_mask = batch_attention_mask.cuda()
batch_token_type_ids = batch_token_type_ids.cuda()
logits1, logits2, logits3 = model(batch_input_ids, batch_attention_mask, batch_token_type_ids)
 test_prob1[start:end, :batch_max_seq_len] += F.softmax(logits1,dim=1 ).cpu().data.numpy()
test_prob2[start:end, :batch_max_seq_len] += F.softmax(logits2,dim=1 ).cpu().data.numpy()
test_prob3[start:end] += F.softmax(logits3,dim=1 ).cpu().data.numpy()
list_words_to_tokens_index += batch_words_to_tokens_index
list_offset += batch_offset
test_word_prob1 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 test_word_prob2 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 for i in range(len(id_candidate_list_sorted)) :
for j in range(len(list_words_to_tokens_index[i])) :
test_word_prob1[i,j] = test_prob1[i, list_words_to_tokens_index[i][j]+list_offset[i]]
test_word_prob2[i,j] = test_prob2[i, list_words_to_tokens_index[i][j]+list_offset[i]]
return test_word_prob1, test_word_prob2, test_prob3
dset = ConcatDataset([ClassDataset(filename, nrows=NROWS)for filename in filenames])
 val_dset = ConcatDataset([ClassDataset(filename, nrows=VAL_NROWS, skiprows=NROWS)for filename in filenames] )<load_pretrained>
start_time = time.time()
data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted = reduce1(n_candidate=10, th_candidate=0.2)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
id_candidate_list_sorted = reduce2(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, n_candidate=4, th_candidate=0.35)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
word_len = 360
start_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 end_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 start_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 end_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 class_prob = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 print("before bert predicting")
 weight_albert = 0.7
model_dir = '.. /models/models/bert/'
test_prob1, test_prob2, test_prob3 = bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob +=(1-weight_albert)*test_prob1
end_prob +=(1-weight_albert)*test_prob2
class_prob +=(1-weight_albert)*test_prob3
print("before albert predicting")
 model_dir = '.. /models/models/albert/'
test_prob1, test_prob2, test_prob3 = albert_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob += weight_albert*test_prob1
end_prob += weight_albert*test_prob2
class_prob += weight_albert*test_prob3
start_label = np.argmax(start_prob, axis=1)
 end_label = np.argmax(end_prob, axis=1)
 temp_dict = {}
for doc_id in id_list:
temp_dict[doc_id] = {
'long_answer': {'start_token': -1, 'end_token': -1},
'long_answer_score': -1.0,
'short_answers': [{'start_token': -1, 'end_token': -1}],
'short_answers_score': -1.0,
'yes_no_answer': 'NONE'
}
for i,(doc_id, candidate_index)in tqdm(enumerate(id_candidate_list_sorted)) :
long_answer_score = 1.0 - class_prob[i,0]
if long_answer_score > temp_dict[doc_id]['long_answer_score']:
temp_dict[doc_id]['long_answer_score'] = long_answer_score
temp_dict[doc_id]['long_answer']['start_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
temp_dict[doc_id]['long_answer']['end_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['end_token']
short_answer_score = 1.0 - class_prob[i,0] - class_prob[i,1]
temp_dict[doc_id]['short_answers_score'] = short_answer_score
temp_dict[doc_id]['short_answers'][0]['start_token'] = -1
temp_dict[doc_id]['short_answers'][0]['end_token'] = -1
temp_dict[doc_id]['yes_no_answer'] = 'NONE'
if max([class_prob[i,3], class_prob[i,4]])> class_prob[i,2]:
if class_prob[i,3] > class_prob[i,4]:
temp_dict[doc_id]['yes_no_answer'] = 'NO'
else:
temp_dict[doc_id]['yes_no_answer'] = 'YES'
else:
short_start_word = int(start_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
short_end_word = int(end_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
if short_end_word > short_start_word:
temp_dict[doc_id]['short_answers'][0]['start_token'] = short_start_word
temp_dict[doc_id]['short_answers'][0]['end_token'] = short_end_word
final_dict = {}
final_dict['predictions'] = []
for doc_id in id_list:
prediction_dict = {
'example_id': doc_id,
'long_answer': {'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['long_answer']['start_token'], 'end_token': temp_dict[doc_id]['long_answer']['end_token']},
'long_answer_score': temp_dict[doc_id]['long_answer_score'],
'short_answers': [{'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['short_answers'][0]['start_token'], 'end_token': temp_dict[doc_id]['short_answers'][0]['end_token']}],
'short_answers_score': temp_dict[doc_id]['short_answers_score'],
'yes_no_answer': temp_dict[doc_id]['yes_no_answer']
}
final_dict['predictions'].append(prediction_dict)
 print("--- %s seconds ---" %(time.time() - start_time))
dloader = DataLoader(dset, batch_size=128, shuffle=True, num_workers=2)
 val_dloader = DataLoader(val_dset, batch_size=128, num_workers=2, shuffle=False )<choose_model_class>
path='/kaggle/input/models/models'
os.chdir(path)
 answers_df = pd.DataFrame.from_dict(final_dict)
 long_best_threshold = 0.65
short_best_threshold = 0.60
def df_long_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['long_answer_score'] > long_best_threshold:
index['start'] = df["predictions"][i]['long_answer']['start_token']
index['end'] = df["predictions"][i]['long_answer']['end_token']
index['score'] = df["predictions"][i]['long_answer_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_short_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['short_answers_score'] > short_best_threshold and df["predictions"][i]['short_answers'][0]['start_token'] != -1:
index['start'] = df["predictions"][i]['short_answers'][0]['start_token']
index['end'] = df["predictions"][i]['short_answers'][0]['end_token']
index['score'] = df["predictions"][i]['short_answers_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_example_id(df):
return df['example_id']
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':' + str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
class Conv(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):
super().__init__()
self.layers = nn.Sequential(
 nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, stride=stride, padding=padding, bias=False),
nn.BatchNorm2d(out_channel),
nn.ReLU(inplace=True)
)
 def forward(self, input):
return self.layers(input)
 class Conv_dw_Conv(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):
super().__init__()
self.layers = nn.Sequential(
 nn.Conv2d(in_channel, in_channel, kernel_size, stride, padding, bias=False, groups=in_channel),
nn.BatchNorm2d(in_channel),
nn.ReLU(inplace=True),
Conv(in_channel, out_channel, kernel_size=1, stride=1, padding=0)
)
 def forward(self, input):
return self.layers(input)
 class MobileNet(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.num_classes = num_classes
self.model = nn.Sequential(
 Conv(3, 32, stride=2),
Conv_dw_Conv(32, 64, kernel_size=3, stride=1),
Conv_dw_Conv(64, 128, kernel_size=3, stride=2),
Conv_dw_Conv(128, 128, kernel_size=3, stride=1),
Conv_dw_Conv(128, 256, kernel_size=3, stride=2),
Conv_dw_Conv(256, 256, kernel_size=3, stride=1),
Conv_dw_Conv(256, 512, kernel_size=3, stride=2),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 1024, kernel_size=3, stride=2),
Conv_dw_Conv(1024, 1024, kernel_size=3, stride=1)
)
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(1024, num_classes)
 def forward(self, input):
x = self.model(input)
 x = self.avg_pool(x)
 x = x.view(-1, 1024)
 out = self.fc(x)
 return out<compute_test_metric>
answers_df['long_indexes_and_scores'] = df_long_index_score(answers_df)
 answers_df['short_indexes_and_scores'] = df_short_index_score(answers_df)
 answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(
 create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(
 create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df = answers_df.drop(
 ['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 sample_submission = pd.read_csv(
 ".. /.. /.. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains(
 "_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains(
 "_short"), "PredictionString"] = short_prediction_strings
def apk(actual, predicted, k=3):

 if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
for i,p in enumerate(predicted):
if p == actual:
score = 1 /(i+1.0)
 return score
def mapk(actual, predicted, k=3):

 return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)] )<train_model>
torch.cuda.empty_cache()
def validation_score(model, val_data_loader):
model.eval()
sum_score = 0
count = 0
for images, labels in val_data_loader:
images = images.to(device)
 labels = labels.to(device)
 batch_size = images.size(0)
 output = model(images)
 topk = output.detach().topk(3, dim=1)[1]
sum_score += mapk(labels.cpu().numpy() , topk.cpu().numpy())* batch_size
count += batch_size
model.train()
return sum_score / count 
<choose_model_class>
del BertTokenizer, BertModel, BertPreTrainedModel, BertConfig, AlbertTokenizer, AlbertModel, AlbertPreTrainedModel, AlbertConfig
model = MobileNet(NUM_CLASSES ).to(device)
 criterion = nn.CrossEntropyLoss().to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.002)
 scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[60000, 130000, 160000, 190000], gamma=0.5 )<train_on_grid>
%%time
print("Start training...")
 epochs = 8
print_every = 1000
validate_every = 10000
model.train()
best_val_score = 0
curr_iter = 0
avg_loss = 0
avg_score = 0
for epoch in range(epochs):
for ix,(images, labels)in enumerate(dloader):
images = images.to(device)
 labels = labels.to(device)
 model.zero_grad()
output = model(images)
 loss = criterion(output, labels)
 loss.backward()
optimizer.step()
avg_loss += loss.item()
topk = output.detach().topk(3, dim=1)[1]
avg_score += mapk(labels.cpu().numpy() , topk.cpu().numpy())
 curr_iter += 1
if(curr_iter)% print_every == 0:
print('Epoch {}, Iteration {} - Train Loss: {:.4f}, MAP@3: {:.3f}'.format(epoch + 1, curr_iter, avg_loss/print_every, avg_score/print_every))
avg_loss = 0
avg_score = 0
if curr_iter % validate_every == 0:
val_score = validation_score(model, val_dloader)
 print('Epoch {}, Iteration {}: validation map@3: {}'.format(epoch + 1, curr_iter, val_score))
if val_score > best_val_score:
print('New best validation score: {}, saving model...'.format(val_score))
best_val_score = val_score
torch.save(model.state_dict() , 'model_checkpoint_best_val.ckpt')
<save_model>


logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',
torch.save(model.state_dict() , 'model_checkpoint.ckpt' )<load_from_csv>
path = '/kaggle/working'
os.chdir(path)
 sample_submission.to_csv('submission2.csv', index=False )
test_dset = ClassDataset('.. /input/test_simplified.csv', mode='test', nrows=None)
 test_dloader = DataLoader(test_dset, batch_size=128, shuffle=False, num_workers=0)
 model.eval()
labels = []
for images in tqdm.tqdm(test_dloader):
images = images.to(device)
 output = model(images)
 _, pred = output.topk(3, 1)
 for i in range(len(images)) :
labels.append(' '.join([decode_dict[pred[i][j].item() ]for j in range(3)])) 
<load_from_csv>
submission_no2 = pd.read_csv("submission1.csv")
 count = 0
for i in range(len(submission_no2)) :
if ':' not in str(submission_no2["PredictionString"][i])and ':' in str(sample_submission["PredictionString"][i]):
count += 1
submission_no2["PredictionString"][i] = sample_submission["PredictionString"][i]
print('count:', count )
%%time
submission = pd.read_csv('.. /input/test_simplified.csv', index_col='key_id' ,usecols=['key_id'])
 print(len(submission))
submission['word'] = labels 
<save_to_csv>
path = 'submission1.csv'
if os.path.exists(path):
os.remove(path)
 path = 'submission2.csv'
if os.path.exists(path):
os.remove(path )
<save_to_csv><EOS> 
submission_no2.to_csv('submission.csv', index=False)