Kaggle competition: Tabular Playground Series - Jan 2021
model_path = list(map(lambda x: os.path.join(PATH, x), model_list))<train_model>
mpl.rcParams['agg.path.chunksize'] = 10000
models = []
for path in model_path:
model = torch.load(path, map_location= 'cuda')
 model.float()
model.eval()
model.to('cuda')
 models.append(model)
 del model<define_variables>
train_data = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test_data = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 print("successfully loaded!" )
sz = 512
test_path = '.. /input/hubmap-kidney-segmentation/test/'
for step, person_idx in enumerate(test_files):
print(f'load {step+1}/{len(test_files)} data...')
 img = tiff.imread(test_path + person_idx + '.tiff' ).squeeze()
if img.shape[0] == 3:
img = img.transpose(1,2,0)
 predict_mask_l1 = np.zeros(( img.shape[0], img.shape[1]), dtype = bool)
 landscape =img.shape[0]// 512
portrait = img.shape[1]// 512
sz = 512
print('predict mask...')
 for x in tqdm(range(landscape)) :
for y in range(portrait):
start_x =(512)* x
end_x =(1024)+ start_x
start_y =(512)* y
end_y =(1024)+ start_y
if x == landscape-1:
start_x = img.shape[0] - 1024
end_x = img.shape[0]
if y == portrait-1:
start_y = img.shape[1] - 1024
end_y = img.shape[1]
sample_img = img[start_x : end_x, start_y : end_y,:]
sample_img = cv2.resize(sample_img,(sz,sz),interpolation = cv2.INTER_AREA)/256
sample_img = torch.cuda.FloatTensor(sample_img.transpose([2,0,1])[np.newaxis,...])
 with torch.no_grad() :
sample_pred = models[0].predict(sample_img ).cpu().numpy() [0,0,:,:]
sample_pred = cv2.resize(sample_pred,(1024,1024),interpolation = cv2.INTER_NEAREST)
 sample_pred = np.where(sample_pred > 0.1, True, False ).astype(bool)
 predict_mask_l1[start_x + 256 : end_x - 256, start_y + 256 : end_y - 256] = sample_pred[256:256 + 512,256:256 + 512]
del sample_img
del sample_pred
gc.collect()
predict_mask_l1 = predict_mask_l1.astype(np.uint8)
 contours, hierarchy = cv2.findContours(predict_mask_l1, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
 predict_mask_l2 = np.zeros(( img.shape[0], img.shape[1]), dtype = bool)
 for cont in tqdm(contours):
center_y, center_x = cont.mean(axis = 0 ).round(0 ).astype(int)[0]
left_x = int(center_x - 512)
 top_y = int(center_y - 512)
 if left_x < 0:
left_x = 0
elif left_x + 1024 > img.shape[0]:
left_x = img.shape[0] - 1024
if top_y < 0:
top_y = 0
elif top_y + 1024 > img.shape[1]:
top_y = img.shape[1] - 1024
sample_img_l2 = img[left_x : left_x + 1024, top_y : top_y+ 1024,:]
sample_img_l2 = cv2.resize(sample_img_l2,(sz,sz),interpolation = cv2.INTER_AREA)/256
sample_img_l2 = torch.cuda.FloatTensor(sample_img_l2.transpose([2,0,1])[np.newaxis,...])
 sample_pred_l2 = None
with torch.no_grad() :
for model in models:
pred_l2 = model.predict(sample_img_l2 ).cpu().numpy() [0,0,:,:]
if sample_pred_l2 is None: sample_pred_l2 = pred_l2
sample_pred_l2 += pred_l2
sample_pred_l2 /= len(models)
 sample_pred_l2 = cv2.resize(sample_pred_l2,(1024,1024),interpolation = cv2.INTER_NEAREST)
 sample_pred_l2 = np.where(sample_pred_l2 > 0.3, True, False ).astype(np.uint8)
 contours_l2, hierarchy = cv2.findContours(sample_pred_l2,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
 if len(contours_l2)< 1:
print('no cotour')
 continue
for cont_l2 in contours_l2:
area = cv2.contourArea(cont_l2)
 min_y, min_x = cont_l2.min(axis = 0 ).round(0 ).astype(int)[0]
max_y, max_x = cont_l2.max(axis = 0 ).round(0 ).astype(int)[0]
if(min_x < 512)and(max_x > 512):
if(min_y < 512)and(max_y > 512):
sample_mask_l2 = np.zeros(sample_pred_l2.shape,
dtype = np.uint8)
 sample_center = cv2.drawContours(sample_mask_l2,
[cont_l2],
0, 
(255, 255, 255),
-1)
 predict_mask_l2[left_x : left_x + 1024,
top_y : top_y+ 1024] =\
np.logical_or(predict_mask_l2[left_x : left_x + 1024,
top_y : top_y+ 1024],
sample_center)
 del predict_mask_l1
del img
gc.collect()
print('convert mask to rle
')
 predict_rle = rle_encode_less_memory(predict_mask_l2)
 sample_submission.loc[person_idx,'predicted'] = predict_rle
del predict_rle
del predict_mask_l2
gc.collect()
sample_submission.reset_index().to_csv('/kaggle/working/submission.csv',index=False)
 sample_submission.reset_index().to_csv('submission.csv', index = False)
 sample_submission = sample_submission.reset_index()
sample_submission.to_csv('/kaggle/working/submission.csv',index=False)
 sample_submission.to_csv('submission.csv', index = False)
 sample_submission<load_from_csv>
outlier = train_data.loc[train_data.target < 1.0]
print(outlier )
sample_sub = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv', index_col='id')
 submission = pd.read_csv('.. /input/my-csv-outputs/d488c759a_single_mask.csv', index_col='id' )<save_to_csv>
train_data.drop([170514], inplace = True )
pub_ids = submission.index.values
predictions = submission.values
sample_sub.loc[pub_ids] = predictions
sample_sub.to_csv('submission.csv')
<load_pretrained>
y_train = train_data["target"]
train_data.drop(columns = ["target"], inplace = True )
df = pd.read_pickle('.. /input/preprocessingdata/df.pkl' )<prepare_x_and_y>
params = { 'n_estimators' : [1500, 2000, 2500],
'learning_rate' : [0.01, 0.02]
}
xgb = XGBRegressor(
 objective = 'reg:squarederror',
subsample = 0.8,
colsample_bytree = 0.8,
learning_rate = 0.01,
tree_method = 'gpu_hist')
 grid_search = GridSearchCV(xgb,
param_grid = params,
scoring = 'neg_root_mean_squared_error',
n_jobs = -1,
verbose = 10)
 grid_search.fit(train_data, y_train)
 print('
Best estimator:')
 print(grid_search.best_estimator_)
 print('
Best score:')
 print(grid_search.best_score_)
 print('
Best hyperparameters:')
 print(grid_search.best_params_ )
X_train = df[df.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = df[df.date_block_num < 33]['item_cnt_month']
X_valid = df[df.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = df[df.date_block_num == 33]['item_cnt_month']
X_test = df[df.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del df<create_dataframe>
clf = XGBRegressor(
 objective = 'reg:squarederror',
subsample = 0.8,
learning_rate = 0.02,
max_depth = 7,
n_estimators = 2500,
tree_method = 'gpu_hist')
 clf.fit(train_data, y_train)
 y_pred_xgb = clf.predict(test_data)
 print(y_pred_xgb )
feature_name = X_train.columns.tolist()
feature_name_indexes = [
'country_part',
'item_category_common',
'item_category_code',
'city_code',
]
def objective(trial):
lgb_train = lgb.Dataset(X_train[feature_name], Y_train)
 lgb_eval = lgb.Dataset(X_valid[feature_name], Y_valid, reference=lgb_train)
 params = {
'objective': 'rmse',
'metric': 'rmse',
'num_leaves': trial.suggest_int('num_leaves', 1000, 1500),
'min_data_in_leaf':10,
'feature_fraction':trial.suggest_uniform('feature_fraction', 0.6, 0.8),
'learning_rate': trial.suggest_uniform('feature_fraction', 0.01, 0.015),
'num_rounds': 1000,
'early_stopping_rounds': 30,
'seed': 1
}
evals_result = {}
gbm = lgb.train(
 params,
lgb_train,
num_boost_round=3000,
valid_sets=(lgb_train, lgb_eval),
feature_name = feature_name,
categorical_feature = feature_name_indexes,
verbose_eval=50,
evals_result = evals_result, 
)
 preds = gbm.predict(X_valid)
 loss = sklearn.metrics.mean_squared_error(Y_valid, preds)
 return loss 
<find_best_params>
solution = pd.DataFrame({"id":test_data.id, "target":y_pred_xgb})
 solution.to_csv("solution.csv", index = False)
 print("saved successful!" )