Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_pretrained>
!pip install timm==0.1.26
eli5.show_weights(perm, top=50, feature_names=top_columns )<save_to_csv>
sys.path.insert(0, ".. /input/efficientdet-torch")
 sys.path.insert(0, ".. /input/omegaconf")
 sys.path.insert(0, ".. /input/weightedboxesfusion")
 warnings.filterwarnings('ignore' )
submission['time_to_failure'] = result_dict_lgb['prediction']
submission.to_csv('submission_eli5.csv' )<init_hyperparams>
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
params = {'num_leaves': 32,
'min_child_samples': 79,
'objective': 'gamma',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"subsample_freq": 5,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501,
'colsample_bytree': 1.0
}<define_variables>
image_dir = ".. /input/vinbigdata-chest-xray-abnormalities-detection/test"
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
dicom = pydicom.dcmread(f"{image_dir}/{image_id}.dicom")
 image = dicom.pixel_array
if "PhotometricInterpretation" in dicom:
if dicom.PhotometricInterpretation == "MONOCHROME1":
image = np.amax(image)- image
image = np.stack([image, image, image])
 image = image.astype('float32')
 image = image - image.min()
image = image / image.max()
image = image.transpose(1, 2, 0)
 if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
scores_dict = {'f_classif': [2.0746468465171377, 2.0753843541953687, 2.062191535440333, 2.0654327826583034, 2.0643551320704936, 2.0617560048382675,
2.061565197738015, 2.0598878198917494, 2.0654865223333143, 2.0632788555735777, 2.058002635080971, 2.051075689018734,
2.0472543961304583, 2.052401474353084, 2.055924154798443, 2.0561794619762352, 2.0549680611994963, 2.057123777802326,
2.0591868861136904, 2.0577745274024553],
'mutual_info_classif': [2.0866763775014006, 2.0745431497064324, 2.0564324832516427, 2.060125564781158, 2.067334544167612, 2.0665943783246448,
2.063891669849029, 2.070194051004794, 2.0667490707700447, 2.0681653852378354, 2.0592743636982345, 2.061260741522344,
2.05680667824411, 2.0565047875243003, 2.058252567141659, 2.0554927194831922, 2.0562776429736873, 2.0618179277444084,
2.06364125584214, 2.0577745274024553],
'n_features': [98, 196, 294, 392, 490, 588, 685, 783, 881, 979, 1077, 1175, 1273, 1370, 1468, 1566, 1664, 1762, 1860, 1958]}<define_variables>
image_ids = glob(os.path.join('.. /input/vinbigdata-chest-xray-abnormalities-detection/test', "*.dicom"))
image_ids = [image_id.split('/')[-1].split('.')[0] for image_id in image_ids]
dataset = DatasetRetriever(image_ids = np.array(image_ids), transforms = get_valid_transforms())
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(dataset, batch_size=8, shuffle=False, num_workers=2, drop_last=False, collate_fn=collate_fn )
scores_dict = {'f_classif': [2.1495892622081354, 2.0778182269587147, 2.0716153738740006, 2.06152950679902, 2.0645162758752553, 2.0627705797004032, 2.0610992303725157,
2.057762113735462, 2.0618360883613627, 2.0603197111525984, 2.06081274633874, 2.0580767195278056, 2.0527646572747127, 2.0498353445032533,
2.052442594925, 2.0564456881902133, 2.0582284644115365, 2.0558612960548635, 2.0580900016350094, 2.058218782401599],
'mutual_info_classif': [2.1235703196243687, 2.084958198672301, 2.0596822478390955, 2.053305869981444, 2.063468853227225, 2.0674399950434323, 2.0658618511287874,
2.063003703200445, 2.0653174905858664, 2.0644340327023656, 2.0748993062333523, 2.0587602096358113, 2.0601495560836076, 2.0559629138548603,
2.0553852701221134, 2.058022171415446, 2.060755947658241, 2.057916705462307, 2.056245795262636, 2.0580691870837056],
'n_features': [10, 110, 210, 310, 410, 510, 610, 710, 810, 910, 1010, 1110, 1210, 1310, 1410, 1510, 1610, 1710, 1810, 1910]}<train_model>
device = torch.device('cuda:0')if torch.cuda.is_available() else torch.device('cpu')
 def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d4')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 14
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 net = DetBenchEval(net, config)
 net.eval()
return net.to(device)
 net = load_net('.. /input/effdet-weights/last-checkpoint.bin' )
corr_matrix = X.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.99)]
X = X.drop(to_drop, axis=1)
 X_test = X_test.drop(to_drop, axis=1)
 result_dict_lgb_lgb = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True )<save_to_csv>
def make_predictions(images, score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
det = net(images, image_scales=torch.tensor([1] * images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:, :4]
scores = det[i].detach().cpu().numpy() [:, 4]
labels = det[i].detach().cpu().numpy() [:, 5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
'labels': labels[indexes]
})
 return [predictions]
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.5, skip_box_thr=0.4):
boxes = [(prediction[image_index]['boxes'] /(image_size - 1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [prediction[image_index]['labels'].tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr,
skip_box_thr=skip_box_thr)
 boxes = boxes *(image_size - 1)
 return boxes, scores, labels
submission['time_to_failure'] = result_dict_lgb['prediction']
submission.to_csv('submission_no_corr.csv' )<define_search_space>
mapping = {0: 'Aortic enlargement', 1: 'Atelectasis', 2: 'Calcification', 3: 'Cardiomegaly', 4: 'Consolidation', 5: 'ILD',6: 'Infiltration', 7: 'Lung Opacity',
8: 'Nodule/Mass', 9: 'Other lesion', 10: 'Pleural effusion', 11: 'Pleural thickening',12: 'Pneumothorax', 13: 'Pulmonary fibrosis'}
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 1
thickness = 3
color =(1, 0, 0)
 for j,(images, image_ids)in enumerate(data_loader):
if j >= 20:
break
predictions = make_predictions(images=images, score_threshold=0.5)
 i = 1
sample = images[i].permute(1, 2, 0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 if len(labels)> 0:
boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 fig, ax = plt.subplots(1, 1, figsize=(8, 8))
for score, box, label in zip(scores, boxes, labels):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(0, 1, 1), thickness)
 cv2.putText(sample, mapping[label] + "---" + str(round(score, 2)) ,(box[0], box[1]),
font, fontScale, color, thickness, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample)
 plt.show()
scores_dict = {'rfe_score': [2.103586938061856, 2.052535910798748, 2.053228199447811], 'n_features': [10, 110, 210]}<train_on_grid>
def format_prediction_string(boxes, scores, labels):
pred_strings = []
for pred in zip(labels, scores, boxes):
pred_strings.append("{0} {1:.4f} {2} {3} {4} {5}".format(int(pred[0]), pred[1], pred[2][0], pred[2][1], pred[2][2], pred[2][3]))
if len(pred_strings)== 0:
return str("14 1 0 0 1 1")
 return " ".join(pred_strings )
%%time
rfr = RandomForestRegressor()
rfr = RandomForestRegressor(n_estimators=50, max_depth=5)
 result_dict_rfr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=rfr)
<train_model>
results = []
for images, image_ids in tqdm(data_loader):
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes * 2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores, labels)
 }
results.append(result )
%%time
linreg = linear_model.LinearRegression(normalize=False, copy_X=True, n_jobs=-1)
 result_dict_linreg = artgor_utils.train_model_regression(X, X_test, y, params=None, folds=folds, model_type='sklearn', model=linreg)
<train_on_grid>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )