Kaggle competition: IEEE's Signal Processing Society - Camera Model Identification
test_test_loader()<load_pretrained>
input_path = Path('.. /input')
 train_path = input_path / 'train'
test_path = input_path / 'test'
test_train_loader()<load_pretrained>
cameras = os.listdir(train_path)
 train_images = []
for camera in cameras:
for fname in sorted(os.listdir(train_path / camera)) :
train_images.append(( camera, fname))
train = pd.DataFrame(train_images, columns=['camera', 'fname'])
 print(train.shape )
class QuestModel(nn.Module):
def __init__(self, model_type="xlnet-base-cased", tokenizer=None, n_classes=30, hidden_layers=[-1, -3, -5, -7, -9]):
super(QuestModel, self ).__init__()
self.model_name = 'QuestModel'
self.model_type = model_type
self.hidden_layers = hidden_layers
if model_type == "bert-base-uncased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
if n_classes == 30:
bert_config.hidden_dropout_prob = 0.1
else:
bert_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "bert-base-cased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-cased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "xlnet-base-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-base-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "xlnet-large-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-large-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "roberta-base":
roberta_model_config = '.. /input/roberta-transformers-pytorch/roberta-base/config.json'
roberta_config = RobertaConfig.from_json_file(roberta_model_config)
 roberta_config.output_hidden_states = True
roberta_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/roberta-transformers-pytorch/roberta-base/pytorch_model.bin')
 self.roberta_model = RobertaModel.from_pretrained(model_path, config=roberta_config)
 self.roberta_model.resize_token_embeddings(len(tokenizer))
if model_type == "bert-base-uncased":
self.hidden_size = 768
elif model_type == "bert-large-uncased":
self.hidden_size = 1024
elif model_type == "bert-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-large-cased":
self.hidden_size = 1024
elif model_type == "roberta-base":
self.hidden_size = 768
else:
raise NotImplementedError
self.fc_1 = nn.Linear(self.hidden_size * len(hidden_layers), self.hidden_size)
 self.fc = nn.Linear(self.hidden_size, n_classes)
 self.selu = nn.SELU()
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 def forward(self, ids, seg_ids):
attention_mask =(ids > 0)
 if(( self.model_type == "bert-base-uncased")\
or(self.model_type == "bert-base-cased")\
or(self.model_type == "bert-large-uncased")\
or(self.model_type == "bert-large-cased")) :
outputs = self.bert_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(( self.model_type == "xlnet-base-cased")\
or(self.model_type == "xlnet-large-cased")) :
attention_mask = attention_mask.float()
outputs = self.xlnet_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[1]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(self.model_type == "roberta-base"):
attention_mask = attention_mask.float()
outputs = self.roberta_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
for j, dropout in enumerate(self.dropouts):
if j == 0:
logit = self.fc(dropout(h))
else:
logit += self.fc(dropout(h))
return logit / len(self.dropouts)
 def test_model(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9]):
x = torch.tensor([[1,2,3,4,5, 0, 0], [1,2,3,4,5, 0, 0]])
 seg_ids = torch.tensor([[0,0,0,0,0, 0, 0], [0,0,0,0,0, 0, 0]])
 model = QuestModel(model_type=model_type, hidden_layers=hidden_layers)
 y = model(x, seg_ids)
 print(y )<train_model>
test_images = []
for fname in sorted(os.listdir(test_path)) :
test_images.append(fname)
 test = pd.DataFrame(test_images, columns=['fname'])
 print(test.shape )
test_model(model_type="bert-base-cased", hidden_layers=[-3, -4, -5, -6, -7] )<load_pretrained>
def get_center_crop(img, d=250):
cy = img.shape[0] // 2
cx = img.shape[1] // 2
return img[cy - d:cy + d, cx - d:cx + d]
def create_bert_base_uncased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-uncased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertuncasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugquestionanswerswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=21, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=9, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=30, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabaseaugdiffswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_question_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_answer_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models<find_best_params>
n_components = 5
pca = PCA(n_components=n_components, svd_solver='randomized',
whiten=True)
 random_images = train.sample(300)
 random_images[:5]
def predict(models, test_loader):
all_scores = []
with torch.no_grad() :
for ids, seg_ids in tqdm(test_loader, total=test_loader.num // test_loader.batch_size):
ids, seg_ids = ids.cuda() , seg_ids.cuda()
scores = []
for model in models:
model = model.cuda()
outputs = torch.sigmoid(model(ids, seg_ids)).cpu()
scores.append(outputs)
 all_scores.append(torch.mean(torch.stack(scores), 0))
all_scores = torch.cat(all_scores, 0 ).numpy()
return all_scores<load_pretrained>
img_set_reds = []
for i, r in random_images.iterrows() :
x = get_center_crop(cv2.imread(".. /input/train/" + r['camera'] + '/' + r['fname']))
img_set_reds.append(np.ravel(x))
img_set_reds = np.asarray(img_set_reds)
 print(img_set_reds.shape)
 print([img_set_reds[i].shape for i in range(10)])
 pf = pca.fit(np.asarray(img_set_reds))
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Question_Answer", batch_size=32 )<predict_on_test>
def get_pca_features(img):
img = np.ravel(img ).reshape(1, -1)
 return pf.transform(img )
roberta_base_models = create_roberta_base_models(tokenizer)
 roberta_base_preds = predict(roberta_base_models, test_loader )<set_options>
t = get_pca_features(get_center_crop(
 cv2.imread(".. /input/train/" + r['camera'] + '/' + r['fname'])) 
)
 t[0]
del roberta_base_models, test_loader
torch.cuda.empty_cache()
gc.collect()<load_pretrained>
cols = ['pca0','pca1', 'pca2',
's1','s2','s3']
for col in cols:
train[col] = None
test[col] = None
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", batch_size=32 )<predict_on_test>
NCORE = 8
color_info = mp.Manager().dict()
q = mp.Queue(maxsize=NCORE)
 iolock = mp.Lock()
pool = mp.Pool(NCORE, initializer=color_stats, initargs=(q, iolock))
for i in train_images:
q.put(i)
 for i in test_images:
q.put(i)
 for _ in range(NCORE):
q.put(None)
 pool.close()
pool.join()
color_info = dict(color_info )
xlnet_base_cased_models = create_xlnet_base_cased_models()
xlnet_base_cased_preds = predict(xlnet_base_cased_models, test_loader )<set_options>
for n, col in enumerate(cols):
train[col] = train['fname'].apply(lambda x: color_info[x][n])
 test[col] = test['fname'].apply(lambda x: color_info[x][n] )
del xlnet_base_cased_models, test_loader
torch.cuda.empty_cache()
gc.collect()<load_pretrained>
y = train['camera'].values
X_train = train[cols].values
X_test = test[cols].values
clf = RandomForestClassifier(n_estimators=5)
 clf.fit(X_train, y )
<predict_on_test><EOS> 
y_pred = clf.predict(X_test)
 subm = pd.read_csv(input_path / 'sample_submission.csv', index_col='fname')
 subm['camera'] = y_pred
subm.to_csv('pca_svm_benchmark.csv' )