Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_from_csv>
warnings.filterwarnings('ignore' )
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))<categorify>
DATA_FOLDER = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection'
EPOCHES = 2
BSIZE = 6
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<define_variables>
ds = VinBigDataDS(DATA_FOLDER)
 loader = D.DataLoader(ds, batch_size=BSIZE, shuffle=True, num_workers=0,
collate_fn=collate_fn )
n_comp_GENES = 100
n_comp_CELLS = 50<concatenate>
def get_model(num_classes):
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)
 in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 return model
g_pca = PCA(n_components=n_comp_GENES, random_state=42)
 train_features = pd.concat(( train_features, pd.DataFrame(g_pca.fit_transform(train_features[GENES]), columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])) , axis=1)
 test_features = pd.concat(( test_features, pd.DataFrame(g_pca.transform(test_features[GENES]), columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])) , axis=1)
 c_pca = PCA(n_components=n_comp_CELLS, random_state=42)
 train_features = pd.concat(( train_features, pd.DataFrame(c_pca.fit_transform(train_features[CELLS]), columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])) , axis=1)
 test_features = pd.concat(( test_features, pd.DataFrame(c_pca.transform(test_features[CELLS]), columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])) , axis=1 )<create_dataframe>
model = get_model(15)
 model = model.to(DEVICE)
 params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.AdamW(params, lr=1e-3 )
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))<merge>
for epoch in range(1,EPOCHES+1):
model.train()
loss_history = { 'loss_classifier': [],
'loss_box_reg': [],
'loss_objectness': [],
'loss_rpn_box_reg':[]}
for images, targets in tqdm(loader):
images = list(image.to(DEVICE)for image in images)
 targets = [{k: v.to(DEVICE)for k, v in t.items() } for t in targets]
loss_dict = model(images, targets)
 losses = sum(loss for loss in loss_dict.values())
 for k, v in loss_dict.items() :
loss_history[k].append(v.item())
 optimizer.zero_grad()
losses.backward()
optimizer.step()
print(f"Loss classifier: {np.array(loss_history['loss_classifier'] ).mean() :6.2f}, \
Loss BoxReg: {np.array(loss_history['loss_box_reg'] ).mean() :6.2f},\
Loss Objectness: {np.array(loss_history['loss_objectness'] ).mean() :6.2f},\
Loss RPN Box Reg: {np.array(loss_history['loss_rpn_box_reg'] ).mean() :6.2f}" )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )<drop_column>
model.eval() ;
filenames = glob.glob(os.path.join(DATA_FOLDER, 'test', '*.dicom'))
outputs = []
for filename in tqdm(filenames):
dfile = dcm.read_file(filename)
 image = dfile.pixel_array
image = historgam(image)
 image = ds.transforms(image.astype(np.float32))
with torch.no_grad() :
prediction = model([image.to(DEVICE)])[0]
line = ''
for i, label in enumerate(prediction['labels']):
if(label < 14)&(prediction['scores'][i].cpu().item() >0.4):
line += str(label.cpu().item())+ ' ' + str(round(prediction['scores'][i].cpu().item() , 4)) + ' '
line += ' '.join([str(round(val.item())) for val in prediction['boxes'][i]])+ ' '
if not line:
line = '14 1.0 0 0 1 1'
outputs.append([filename.split('/')[-1][:-6], line] )
<define_variables><EOS> 
submission = pd.DataFrame(outputs, columns=['image_id', 'PredictionString'])
 submission.to_csv('submission.csv', index=False )