Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
VERSION = "nightly"
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version $VERSION
def get_train_data() :
filename = os.path.join(data_dir, 'train.csv')
 train = pd.read_csv(filename, index_col=0)
 X_train = train.iloc[:, 1:]
y_train = train['target']
return X_train, y_train
def get_test_data() :
filename = os.path.join(data_dir, 'test.csv')
 test = pd.read_csv(filename, index_col=0)
 return test<train_on_grid>
!pip install transformers==3.0.0
log_clf = LogisticRegression(random_state=42)
 param_grid = {
'class_weight' : ['balanced', None],
'penalty' : ['l2','l1'],
'C' : [0.001, 0.01, 0.1, 1, 10, 100, 1000],
'solver' : ['liblinear', 'saga'],
}
grid = GridSearchCV(estimator=log_clf, cv=5, param_grid=param_grid,
scoring='roc_auc', verbose=1, n_jobs=-1)
 grid.fit(X_train, y_train)
 print("Best Score: {0}".format(grid.best_score_))
print("Best Parameters: {0}".format(grid.best_params_))
best_parameters = grid.best_params_<train_model>
train = pd.read_csv('data/train.csv')
 train = train.fillna('')
 train['text'] = train['keyword'] + ' ' + train['location'] + ' '+ train['text']
train = train[['text','target']]
df_train, df_valid = train_test_split(train, test_size=0.2, random_state=42)
 df_train.to_csv('train.csv')
 df_valid.to_csv('valid.csv' )
log_clf = LogisticRegression(**best_parameters)
 log_clf.fit(X_train, y_train)
 selector = RFE(log_clf, 25, step=1)
 selector.fit(X_train, y_train)
 scores_table(selector, X_train, y_train, 'selector_clf', n_splits=10 )<predict_on_test>
class BertStyleModel(torch.nn.Module):
def __init__(self, model_type):
super().__init__()
self.model_type = model_type
if(model_type == 'roberta'):
config_path = 'roberta-base'
model_path = 'roberta-base'
config = RobertaConfig.from_pretrained(config_path)
 config.output_hidden_states = True
self.bert = RobertaModel.from_pretrained(model_path, config=config)
 elif(model_type == 'distilbert'):
config_path = 'distilbert-base-uncased'
config = DistilBertConfig.from_json_file(config_path)
 model_path = 'distilbert-base-uncased'
config.output_hidden_states = True
self.bert = DistilBertModel.from_pretrained(model_path, config=config)
 elif(model_type == 'bert'):
config_path = 'bert-base-uncased'
config = BertConfig.from_pretrained(config_path)
 config.output_hidden_states = True
model_path = 'bert-base-uncased'
self.bert = BertModel.from_pretrained(model_path, config=config)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.cls_token_head = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(768 * 4, 768),
nn.ReLU(inplace=True), 
)
 self.classifier = nn.Linear(768, 1)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):
if(self.model_type == 'roberta'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
elif(self.model_type == 'distilbert'):
outputs = self.bert(input_ids, attention_mask=attention_mask)
 hidden_layers = outputs[1]
elif(self.model_type == 'bert'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
hidden_states_cls_embeddings = [x[:, 0] for x in hidden_layers[-4:]]
x = torch.cat(hidden_states_cls_embeddings, dim=-1)
 cls_output = self.cls_token_head(x)
 logits = torch.mean(torch.stack([
self.classifier(self.high_dropout(cls_output))
for _ in range(5)
 ], dim=0), dim=0)
 outputs = logits
return outputs
y_test = log_clf.predict_proba(X_test )<save_to_csv>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def loss_fn(ypred, label):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def seed_all(seed):
random.seed(seed)
 torch.manual_seed(seed)
 np.random.seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed_all(seed )
filename = os.path.join(data_dir, 'sample_submission.csv')
 submission = pd.read_csv(filename)
 filename = os.path.join(output_dir, 'submission.csv')
 submission['target'] = y_test
submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
warnings.filterwarnings("ignore")
 class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }
print('Installing packages')
 !pip uninstall -y allennlp torch
!pip install /kaggle/input/torch16/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
!pip install /kaggle/input/pytorch-image-models/
!pip install /kaggle/input/yacs-yet-another-configuration-system/yacs
!pip install /kaggle/input/landmark-lib/
!pip install /kaggle/input/semantic-segmentation-pytorch/
!pip install /kaggle/input/superpoint-pytorch/
print('Installed packages' )<set_options>
mx = BertStyleModel('bert')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "bert.bin")
 np.save('bert', np.array(o))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )
os.environ["LRU_CACHE_CAPACITY"] = "3"
cv2.setNumThreads(0)
 cv2.ocl.setUseOpenCL(False)
 torch.set_grad_enabled(False)
<set_options>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = _run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )
seed = 42
torch.manual_seed(seed)
 np.random.seed(seed)
 random.seed(seed )<import_modules>
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)
 }
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('bert')
 model.load_state_dict(torch.load("bert.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result.npy', ypred)
 return ypred
from superpoint_pytorch.models.superpoint import SuperPoint as SP
from superpoint_pytorch.models.superglue import SuperGlue as SG<load_pretrained>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )
def load_weight(model, weight):
print(f"=> loading checkpoint {weight}")
 ckpt = torch.load(weight, "cpu")
 state_dict = ckpt.pop('state_dict')
 new_state_dict = {}
for k, v in state_dict.items() :
if k.endswith('seesaw_loss.N'):
continue
new_k = k
if new_k.startswith('module.'):
new_k = new_k[7:]
new_state_dict[new_k.replace("se_module", "se")] = v
model.load_state_dict(new_state_dict)
 print('=> loaded state dict' )<define_variables>
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result.npy')
 submit['target'] = ypred
submit['target'].hist()
plt.show()
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv' )<define_variables>
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit.csv', index=False)
NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 5
TOP_K = 3
MAX_INLIER_SCORE = 200
MAX_REPROJECTION_ERROR = 4.0
MAX_RANSAC_ITERATIONS = 1000
HOMOGRAPHY_CONFIDENCE = 0.99<init_hyperparams>
class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }
stride_factor = 1
rf, stride, padding = [291.0, 16.0 * stride_factor, 145.0]
feature_depth = 128
scales =(1,)
 ABS_THRESHOLD = 175
LOCAL_FEATURE_NUM_TENSOR = 1000
NMS_IOU = 1.
 NUM_EMBEDDING_DIMENSIONS = 512<choose_model_class>
mx = BertStyleModel('roberta')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 print("%s: traning"%device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "roberta.bin")
 np.save('roberta', np.array(t))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )
aug = Compose([
Normalize() ,
ToTensorV2() ,
] )<categorify>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = _run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )
def get_model(config_path, weight_path):
cfg = get_cfg()
cfg.merge_from_file(config_path)
 cfg.MODEL.BACKBONE.PRETRAINED = False
INTERP = cfg.DATA.INTERP
class GEMPoolExtractor(GEMPoolCNN):
def __init__(self, cfg):
super(GEMPoolExtractor, self ).__init__(cfg)
 def forward(self, images):
with autocast() :
features = self.backbone(images)
 features = [features[f] for f in self.in_features]
_, global_features = features
global_features = self.pool(global_features)
 return global_features
model = GEMPoolExtractor(cfg)
 load_weight(model, weight_path)
 del model.cls_head; gc.collect()
model.eval()
model = model.cuda()
return model, INTERP
model1, INTERP = get_model('/kaggle/input/landmark-lib/configs/google-landmark/resnext101_32x4d.yaml',
'/kaggle/input/best-resnext101-32x4d-cluster/best_resnext101_32x4d_clusterv2.pth')
 model2, INTERP = get_model('/kaggle/input/best-seresnet101-cluster/seresnet101.yaml',
'/kaggle/input/best-seresnet101-cluster/best_seresnet101_clusterv2.pth')
 model5, INTERP = get_model('/kaggle/input/landmark-lib/configs/google-landmark/resnext101_32x4d.yaml',
'/kaggle/input/best-resnext101-32x4d-cluster/best_resnext101_32x4d_clusterv2_sz736.pth')
 model6, INTERP = get_model('/kaggle/input/best-seresnet101-cluster/seresnet101.yaml',
'/kaggle/input/best-seresnet101-cluster/best_seresnet101_clusterv2_sz736.pth')
 MODELS = [model1, model2,
model5, model6]
MODEL_NAMES = ['resnext101_clusterv2', 'seresnet101_clusterv2',
'resnext101_clusterv2_sz736', 'seresnet101_clusterv2_sz736']
MODEL_WEIGHTS = [1.0] * 4<define_variables>
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
}
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('roberta')
 model.load_state_dict(torch.load("roberta.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result2.npy', ypred)
 return ypred
def extract_global_features(image_root_dir, pre_compute=False):
image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 print("Num images: ", num_embeddings)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( len(MODELS), num_embeddings, NUM_EMBEDDING_DIMENSIONS))
if pre_compute:
for m, name in enumerate(MODEL_NAMES):
emb_dict = torch.load(f"/kaggle/input/train-embeddings/{name}.pth", "cpu")
 for i, image_path in tqdm(enumerate(image_paths), total=num_embeddings):
image_id = image_path.stem
if image_id in emb_dict.keys() :
ids[i] = int(image_path.name.split('.')[0], 16)
 embeddings[m, i, :] = emb_dict[image_id].squeeze(0 ).numpy()
del emb_dict; gc.collect()
for i, image_path in tqdm(enumerate(image_paths), total=num_embeddings):
if i >= num_embeddings:
break
if ids[i] != None:
continue
ids[i] = int(image_path.name.split('.')[0], 16)
 image = np.array(Image.open(image_path ).convert('RGB'))
image = thumbnail(image)
 image = aug(image=image)['image']
image.unsqueeze_(0)
 image = image.cuda(non_blocking=True)
 for m in range(len(MODELS)) :
global_descriptors = MODELS[m](image)
 embeddings[m, i, :] = F.normalize(global_descriptors.sum(dim=0), dim=0 ).cpu().numpy()
return ids, embeddings<categorify>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )
def to_hex(image_id):
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(data_dir, image_id):
name = to_hex(image_id)
 return os.path.join(data_dir, name[0], name[1], name[2],
'{}.jpg'.format(name))<compute_test_metric>
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result2.npy')
 submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False)
def get_total_score(num_inliers, global_score):
local_score = min(num_inliers + 1, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score * global_score<find_best_params>
Try to concat 2 model.
 I really think: ROBERTA is better for this task.So try to multiply Roberta result with greatter 1 number.Get result better.
 Summary: roberta*1.3 and get top 15%
local_extractor = SP({
'keypoint_threshold': 0.001,
'max_keypoints': 2048,
"seg_backbone": "hrnetv2",
'seg_mask': True,
'homographic': False,
})
 local_matcher = SG({'match_threshold': 0.2, 'weights': 'outdoor', 'sinkhorn_iterations': 150})
 local_extractor.eval()
local_matcher.eval()
local_extractor = local_extractor.cuda()
local_matcher = local_matcher.cuda()
print('Finished loading SuperPoint + SuperGlue' )<statistical_test>
bert = np.load('result.npy')
 roberta= np.load('result2.npy' )
def get_num_inliers_superglue(data, local_matcher=local_matcher):
scores = local_matcher(data)['matching_scores0']
num_inliers = torch.clamp(scores, 0.0, 1.0 ).sum().item()
return num_inliers<normalization>
bert_ = np.array([int(b>0.5)for b in bert])
 roberta_ = np.array([int(b>0.5)for b in roberta])
HEIGHT, WIDTH = 896, 672
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 def load_image(image_path):
image = np.array(Image.open(image_path ).convert("RGB"))
image = functional.resize(image, height=HEIGHT, width=WIDTH, interpolation=cv2.INTER_LINEAR)
 image = image.astype(np.float32)/ 255.
 image = torch.from_numpy(image ).permute(2, 0, 1)
 return image
image_paths = []
image_paths.append(get_image_path(TEST_IMAGE_DIR, test_image_id))
for i in range(len(train_ids_labels_and_scores)) :
train_image_id, _, _ = train_ids_labels_and_scores[i]
image_paths.append(get_image_path(TRAIN_IMAGE_DIR, train_image_id))
images = torch.stack([load_image(p)for p in image_paths], 0)
 images = images.cuda(non_blocking=True)
 outputs = local_extractor({"image": images})
 data = {}
for k, v in outputs.items() :
data[k+'0'] = v[0][None]
data['image_shape0'] = data['image_shape1'] = [HEIGHT, WIDTH]
for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
for k, v in outputs.items() :
data[k+'1'] = v[i+1][None]
num_inliers = get_num_inliers_superglue(data)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores<load_from_csv>
submit = pd.read_csv('sample_submission.csv')
 ypred = roberta2
submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False )
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap<categorify>
submit = pd.read_csv('/kaggle/input/nlpstarted/submit2(1 ).csv')
 submit.to_csv('submission.csv', index=False )
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
aggregate_scores[label] = aggregate_scores.get(label, 0)+ score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map<predict_on_test>
VERSION = "nightly"
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version $VERSION
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR, pre_compute=True)
 test_embeddings = np.transpose(test_embeddings,(1, 0, 2)).reshape(test_embeddings.shape[0], -1)
 train_embeddings = np.transpose(train_embeddings,(1, 0, 2)).reshape(train_embeddings.shape[0], -1)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
similarities = spatial.distance.cdist(test_embeddings[test_index][np.newaxis,...],
train_embeddings, 'cosine')
 distances = 1 - similarities
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions 
<save_to_csv>
!pip install transformers==3.0.0
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
<categorify>
train = pd.read_csv('data/train.csv')
 train = train.fillna('')
 train['text'] = train['keyword'] + ' ' + train['location'] + ' '+ train['text']
train = train[['text','target']]
df_train, df_valid = train_test_split(train, test_size=0.2, random_state=42)
 df_train.to_csv('train.csv')
 df_valid.to_csv('valid.csv' )
def subsample(x, n=32):
out = dict()
for k in list(x.keys())[:n]:
out[k] = x[k]
return out
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
TEST_IMAGE_DIR = "sample"
TRAIN_IMAGE_DIR = "sample"
_, post_verification_predictions = get_predictions(labelmap)
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
save_submission_csv()
else:
save_submission_csv(post_verification_predictions )<install_modules>
class BertStyleModel(torch.nn.Module):
def __init__(self, model_type):
super().__init__()
self.model_type = model_type
if(model_type == 'roberta'):
config_path = 'roberta-base'
model_path = 'roberta-base'
config = RobertaConfig.from_pretrained(config_path)
 config.output_hidden_states = True
self.bert = RobertaModel.from_pretrained(model_path, config=config)
 elif(model_type == 'distilbert'):
config_path = 'distilbert-base-uncased'
config = DistilBertConfig.from_json_file(config_path)
 model_path = 'distilbert-base-uncased'
config.output_hidden_states = True
self.bert = DistilBertModel.from_pretrained(model_path, config=config)
 elif(model_type == 'bert'):
config_path = 'bert-base-uncased'
config = BertConfig.from_pretrained(config_path)
 config.output_hidden_states = True
model_path = 'bert-base-uncased'
self.bert = BertModel.from_pretrained(model_path, config=config)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.cls_token_head = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(768 * 4, 768),
nn.ReLU(inplace=True), 
)
 self.classifier = nn.Linear(768, 1)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):
if(self.model_type == 'roberta'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
elif(self.model_type == 'distilbert'):
outputs = self.bert(input_ids, attention_mask=attention_mask)
 hidden_layers = outputs[1]
elif(self.model_type == 'bert'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
hidden_states_cls_embeddings = [x[:, 0] for x in hidden_layers[-4:]]
x = torch.cat(hidden_states_cls_embeddings, dim=-1)
 cls_output = self.cls_token_head(x)
 logits = torch.mean(torch.stack([
self.classifier(self.high_dropout(cls_output))
for _ in range(5)
 ], dim=0), dim=0)
 outputs = logits
return outputs
!pip install -U.. /input/kerasapplications/Keras_Applications-1.0.8-py3-none-any.whl
!pip install -U.. /input/efficientnetwhl/efficientnet-1.1.0-py3-none-any.whl 
<set_options>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def loss_fn(ypred, label):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def seed_all(seed):
random.seed(seed)
 torch.manual_seed(seed)
 np.random.seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed_all(seed )
print(tf.__version__)
 tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 mixed_precision = False
DEVICE = "TPU"
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 strategy_num = strategy.num_replicas_in_sync
print(f'strategy.num: {strategy_num}')
 config = {
'learning_rate': 5e-3,
'momentum': 0.9,
'scale': 30,
'margin': 0.3,
'clip_grad': 10.0,
'n_epochs': 3,
'input_size':(800,800, 3),
'n_classes': 81313,
}
BATCH_SIZE=32*strategy_num
AUTO = tf.data.experimental.AUTOTUNE<load_pretrained>
warnings.filterwarnings("ignore")
 class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }
model_b7_1024 = create_model_b7_1024()
model_b7_1024.load_weights('.. /input/glr2-sub-model/Acc-0.994-efnb7-model.h5')
 model_b7_1024.trainable=False
model_b7_1024 = tf.keras.Model(
 inputs=model_b7_1024.get_layer('input/image' ).input,
outputs=model_b7_1024.get_layer('head/bn_0' ).output)
 model_b7_1024.summary()
model_b6_512 = create_model_b6_512()
model_b6_512.load_weights('.. /input/glr2-sub-model/Acc-0.997-effb6-512-model.h5')
 model_b6_512.trainable=False
model_b6_512 = tf.keras.Model(
 inputs=model_b6_512.get_layer('input/image' ).input,
outputs=model_b6_512.get_layer('head/bn_1' ).output)
 model_b6_512.summary()
model_b7_512 = create_model_b7_512()
model_b7_512.load_weights('.. /input/glr2-sub-model/Acc-0.985-efnb7-512-model.h5')
 model_b7_512.trainable=False
model_b7_512 = tf.keras.Model(
 inputs=model_b7_512.get_layer('input/image' ).input,
outputs=model_b7_512.get_layer('head/bn_1' ).output)
 model_b7_512.summary()
model_b7_2560 = create_model_b7_2560()
model_b7_2560.load_weights('.. /input/glr2-sub-model/Acc-0.996-efnb7-2560-model.h5')
 model_b7_2560.trainable=False
model_b7_2560 = tf.keras.Model(
 inputs=model_b7_2560.get_layer('input/image' ).input,
outputs=model_b7_2560.get_layer('head/pooling' ).output)
 model_b7_2560.summary()<load_from_csv>
mx = BertStyleModel('bert')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "bert.bin")
 np.save('bert', np.array(o))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )
def read_test_file() :
files_paths = glob.glob('.. /input/landmark-recognition-2020' + '/test/*/*/*/*')
 mapping = {}
for path in files_paths:
mapping[path.split('/')[-1].split('.')[0]] = path
df = pd.read_csv('.. /input/landmark-recognition-2020/sample_submission.csv')
 df['path'] = df['id'].map(mapping)
 return df
df=read_test_file()
print(len(df))
def load_labelmap() :
with open('.. /input/landmark-recognition-2020/train.csv', mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def read_train_file() :
labelmap=load_labelmap()
files_paths = glob.glob('.. /input/landmark-recognition-2020' + '/train/*/*/*/*')
 mapping = {}
for path in files_paths:
image_id=path.split('/')[-1].split('.')[0]
mapping[path] = image_id
return mapping
df_train=read_train_file()
print(len(df_train))<normalization>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = _run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )
AUTO = tf.data.experimental.AUTOTUNE
def read_image(img_id,image_path):
image = tf.io.read_file(image_path)
 image = tf.image.decode_jpeg(image, channels=3)
 image = tf.image.resize(image, config['input_size'][0:2], method='bilinear')
 image = tf.cast(image, tf.float32)
 image /= 255.
 return image,img_id
def create_dataset(df,batch_size):
image_paths= df.path
img_id=df.id
dataset = tf.data.Dataset.from_tensor_slices(( img_id,image_paths))
dataset = dataset.map(read_image,tf.data.experimental.AUTOTUNE)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(AUTO)
 return dataset
def create_train_dataset(df,batch_size):
image_paths= list(df_train.keys())
 img_id=list(df_train.values())
 dataset = tf.data.Dataset.from_tensor_slices(( img_id,image_paths))
dataset = dataset.map(read_image,tf.data.experimental.AUTOTUNE)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(AUTO)
 return dataset 
<define_variables>
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)
 }
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('bert')
 model.load_state_dict(torch.load("bert.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result.npy', ypred)
 return ypred
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 70000
HOMOGRAPHY_CONFIDENCE = 0.99<normalization>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )
<train_model>
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result.npy')
 submit['target'] = ypred
submit['target'].hist()
plt.show()
@tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])
 def serving_train(input_image):
outputs0 = model_b7_1024(input_image)
 outputs0 = tf.math.l2_normalize(outputs0,axis=-1)
 outputs1 = model_b6_512(input_image)
 outputs1 = tf.math.l2_normalize(outputs1,axis=-1)
 outputs2 = model_b7_512(input_image)
 outputs2 = tf.math.l2_normalize(outputs2,axis=-1)
 outputs3 = model_b7_2560(input_image)
 outputs3 = tf.math.l2_normalize(outputs3,axis=-1)
 outputs=tf.concat([outputs0,outputs1,outputs2,outputs3],axis=-1)
 return outputs
def get_train_ids_labels_and_scores() :
train_dataset=create_train_dataset(df_train,8)
 features_2048=[]
ID_marks=[]
train_dataset = tqdm.tqdm(train_dataset)
 for k, inputs in enumerate(train_dataset):
ID_marks+=list(inputs[1].numpy())
 out=serving_train(inputs[0])
 features_2048.append(out)
 del inputs
del out
gc.collect()
del train_dataset
gc.collect()
features_2048=tf.concat(features_2048,axis=0)
 ID_marks=np.array(ID_marks)
 features_2048=tf.concat(features_2048,axis=0)
 features_2048=tf.transpose(features_2048)
 print(features_2048.shape)
 @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])
 def serving(input_image):
outputs0 = model_b7_1024(input_image)
 outputs0 = tf.math.l2_normalize(outputs0,axis=-1)
 outputs1 = model_b6_512(input_image)
 outputs1 = 0.7*tf.math.l2_normalize(outputs1,axis=-1)
 outputs2 = model_b7_512(input_image)
 outputs2 = tf.math.l2_normalize(outputs2,axis=-1)
 outputs3 = model_b7_2560(input_image)
 outputs3 = tf.math.l2_normalize(outputs3,axis=-1)
 outputs=tf.concat([outputs0,outputs1,outputs2,outputs3],axis=-1)
 outputs=tf.matmul(outputs,features_2048)
 return outputs
num_k=NUM_TO_RERANK
labelmap=load_labelmap()
test_dataset=create_dataset(df,8)
 ID_test=list(df.id)
 test_dataset = tqdm.tqdm(test_dataset)
 train_ids_labels_and_scores=[]
for k, inputs in enumerate(test_dataset):
out=serving(inputs[0] ).numpy()
for j in range(inputs[1].shape[0]):
top_k = np.argpartition(out[j], -num_k)[-num_k:]
top_k_score=out[j][top_k]/4.0
top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]
top_k_label=[labelmap[x] for x in top_k_id]
train_ids_labels_and_scores.append(list(zip(top_k_id,top_k_label,top_k_score)))
 del inputs
del out
gc.collect()
del features_2048
del test_dataset
gc.collect()
return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores<load_pretrained>
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit.csv', index=False)
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'] )<data_type_conversions>
class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }
def get_image_path(subset, image_id):
name =image_id
return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.99):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + 0.8*global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = test_id
aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):
test_ids, train_ids_labels_and_scores=get_train_ids_labels_and_scores()
pre_verification_predictions = get_prediction_map(test_ids, train_ids_labels_and_scores)
 test_ids = tqdm.tqdm(test_ids)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions 
<save_to_csv>
mx = BertStyleModel('roberta')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 print("%s: traning"%device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "roberta.bin")
 np.save('roberta', np.array(t))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )
def save_submission_csv(predictions=None):
if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
if score>0.3:
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 else:
csv_writer.writerow({'id': image_id, 'landmarks': ''})
 def main() :
pass
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 if __name__ == '__main__':
main()<import_modules>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = _run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8500000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<import_modules>
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
}
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('roberta')
 model.load_state_dict(torch.load("roberta.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result2.npy', ypred)
 return ypred
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8500000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<define_variables>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8000000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<load_from_csv>
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result2.npy')
 submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False)
pd.options.display.max_columns = None
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 potential_features = ['ps_ind_06_bin',
'ps_ind_07_bin',
'ps_ind_08_bin',
'ps_ind_16_bin',
'ps_ind_17_bin',
'ps_car_08_cat',
'ps_ind_04_cat',
'ps_car_03_cat',
'ps_car_11_cat',
'ps_car_09_cat',
'ps_car_06_cat',
'ps_ind_05_cat',
'ps_car_05_cat',
'ps_car_04_cat',
'ps_car_01_cat',
'ps_car_02_cat',
'ps_ind_02_cat',
'ps_car_07_cat',
'ps_car_13',
'ps_car_12',
'ps_reg_02',
'ps_reg_03',
'ps_car_15',
'ps_reg_01',
'ps_ind_15',
'ps_ind_01',
'ps_car_14',
'ps_ind_03',
'ps_ind_14']
train_df = train_df[['target'] + potential_features]
test_ids = test_df['id']
test_df = test_df[potential_features]<split>
Try to concat 2 model.
 I really think: ROBERTA is better for this task.So try to multiply Roberta result with greatter 1 number.Get result better.
 Summary: roberta*1.3 and get top 15%
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini<randomize_order>
bert = np.load('result.npy')
 roberta= np.load('result2.npy' )
over_factor = 0.6
row_count = round(len(train_df[train_df['target'] == 1])* over_factor)
 new_rows = train_df[train_df['target'] == 1].sample(row_count, replace=True, random_state=1)
 train_df = train_df.append(new_rows, ignore_index=True)
 train_df['target'].value_counts(1 )<define_variables>
bert_ = np.array([int(b>0.5)for b in bert])
 roberta_ = np.array([int(b>0.5)for b in roberta])
train_miss = []
test_miss = []
print("train columns with missing data: ")
 for col in train_df.columns:
if(train_df[col].min() == -1)and('cat' not in col):
train_miss.append(col)
 print(col)
 print("
test columns with missing data: ")
 for col in test_df.columns:
if(test_df[col].min() == -1)and('cat' not in col):
test_miss.append(col)
 print(col )<train_model>
submit = pd.read_csv('sample_submission.csv')
 ypred = roberta2
submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False )
for col in train_miss:
print("Train: ", col, ": ", len(train_df[train_df[col] == -1]))
for col in test_miss:
print("Test: ", col, ": ", len(test_df[test_df[col] == -1]))<feature_engineering>
submit = pd.read_csv('/kaggle/input/nlpstarted/submit2(1 ).csv')
 submit.to_csv('submission.csv', index=False )