Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<save_to_csv>
import time
import gc
import os
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from numba import jit
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False)
<install_modules>
N_SPLITS = 4
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript'
sys.path.append(package_path )<import_modules>
src_data_path = ".. /input/porto-seguro-safe-driver-prediction"
df_train = pd.read_csv(os.path.join(src_data_path, "train.csv"))
df_test = pd.read_csv(os.path.join(src_data_path, "test.csv"))
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet<categorify>
id_test = df_test['id'].values
target_train = df_train['target'].values
df_train = df_train.drop(['target', 'id'], axis=1)
 df_test = df_test.drop(['id'], axis=1 )
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
incomplete_columns = ['ps_car_03_cat', 'ps_car_05_cat']
df_train = df_train.drop(incomplete_columns, axis=1)
 df_test = df_test.drop(incomplete_columns, axis=1 )
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
df_train = df_train.replace(-1, np.nan)
 df_test = df_test.replace(-1, np.nan )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
cat_features = [a for a in df_train.columns if a.endswith('cat')]
def preprocess_cat_feature(df, cat_feats):
for column in cat_feats:
temp = pd.get_dummies(pd.Series(df[column]), prefix=column)
 df = pd.concat([df, temp],axis=1)
 df = df.drop([column],axis=1)
 return df
df_train = preprocess_cat_feature(df_train, cat_features)
 df_test = preprocess_cat_feature(df_test, cat_features )
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
sorted_columns_by_fi = sorted(zip(model.feature_importances_, X.columns))
col_to_drop = [x[1] for x in sorted_columns_by_fi if x[0] < 100]
df_train = df_train.drop(col_to_drop, axis=1)
 df_test = df_test.drop(col_to_drop, axis=1 )
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<load_pretrained>
class model_builder(object):
def __init__(self, n_splits):
self.n_splits = n_splits
self.stacker = LogisticRegression()
pass
def fit_predict(self, X, y, T, params_list):
y_valid_pred = 0 * y
y_submit_pred = 0
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 stack_train = np.zeros(( X.shape[0], len(params_list)))
 stack_test = np.zeros(( T.shape[0], len(params_list)))
 for param_index, params in enumerate(params_list):
kfold = KFold(n_splits = self.n_splits, random_state = 1, shuffle = True)
 model = lgb.LGBMClassifier(**params)
 stack_test_i = np.zeros(( T.shape[0], self.n_splits))
for fold_index,(train_index, val_index)in enumerate(kfold.split(X)) :
print("model ", param_index, " fold ", fold_index)
 X_train = X[train_index]
y_train = y[train_index]
X_valid = X[val_index]
model.fit(X_train, y_train, verbose=True)
 y_pred = model.predict_proba(X_valid)[:, 1]
stack_train[val_index, param_index] = y_pred
pred = model.predict_proba(T)
 stack_test_i[:, fold_index] = pred[:, 1]
stack_test[:, param_index] = stack_test_i.mean(axis=1)
 self.stacker.fit(stack_train, y)
 pred = self.stacker.predict_proba(stack_test)[:, 1]
return pred
params_list = [
{
'learning_rate' : 0.02,
'n_estimators' : 650,
'max_bin' : 10,
'subsample' : 0.8,
'subsample_freq' : 10,
'colsample_bytree' : 0.8,
'min_child_samples' : 500,
'seed' : 99
},
{
'learning_rate' : 0.02,
'n_estimators' : 1100,
'subsample' : 0.7,
'subsample_freq' : 2,
'colsample_bytree' : 0.3,
'num_leaves' : 16,
'seed' : 99
},
{
'n_estimators' : 1100,
'max_depth' : 4,
'learning_rate' : 0.02,
'seed' : 99
},
]
builder = model_builder(n_splits=N_SPLITS)
 y_test_pred = builder.fit_predict(X=df_train, y=target_train, T=df_test, params_list=params_list )
ckpt_path = ".. /input/unet-starter-kernel-pytorch-lb-0-88/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )<save_to_csv>
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('lgb_submit.csv', float_format='%.6f', index=False )