Kaggle competition: State Farm Distracted Driver Detection
<SOS> metric: MulticlassLoss Kaggle data source: state-farm-distracted-driver-detection<merge>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import keras
import numpy
from keras.preprocessing.image import ImageDataGenerator
sub = sub_prep.merge(submission['ForecastId'], on=['ForecastId'], how='right')
 sub<sort_values>
import keras
import numpy
from keras.preprocessing.image import ImageDataGenerator
sub=sub[['ForecastId','ConfirmedCases','Fatalities']]
sub=sub.sort_values('ForecastId')
 sub<save_to_csv>
train_datagen = ImageDataGenerator(
 rescale=1./255, validation_split=0.2)
sub.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False )<merge>
train_data = '.. /input/state-farm-distracted-driver-detection/train'
test_data = '.. /input/state-farm-distracted-driver-detection/test'
train_generator = train_datagen.flow_from_directory(
 train_data,
target_size=(224, 224),
batch_size=32,
class_mode='categorical',
subset='training')
 val_generator = train_datagen.flow_from_directory(
 train_data,
target_size=(224,224),
batch_size=32,
class_mode='categorical',
subset='validation')
train['Date']=train.Date.astype('datetime64[ns]')
 verify=train[['key','Date','ConfirmedCases','Fatalities']].merge(test[['key','Date','ForecastId']], how='inner', on=['key','Date'])
 pred=verify[['ForecastId']].merge(sub, how='inner', on=['ForecastId'] )<compute_test_metric>
ac_labels= ["c0: safe driving",
"c1: texting - right",
"c2: talking on the phone - right",
"c3: texting - left",
"c4: talking on the phone - left",
"c5: operating the radio",
"c6: drinking",
"c7: reaching behind",
"c8: hair and makeup",
"c9: talking to passenger"]
RMSLE(pred['Fatalities'].values,verify['Fatalities'].values )<compute_test_metric>
imgs, labels = next(train_generator )
RMSLE(pred['ConfirmedCases'].values,verify['ConfirmedCases'].values )<merge>
def list_counts(start_dir):
lst = sorted(os.listdir(start_dir))
out = [(fil, len(os.listdir(os.path.join(start_dir, fil)))) for fil in lst if os.path.isdir(os.path.join(start_dir,fil)) ]
return out
out = list_counts(train_data)
 labels, counts = zip(*out)
 print("Total number of images : ",functools.reduce(lambda a,b : a+b, counts))
out
best_model_key=ConfirmedCases_all_result_Final[['key','best_model']].drop_duplicates()
max_number_current=train.groupby('key' ).max() [['ConfirmedCases','Fatalities']].reset_index()
best_model_key=best_model_key.merge(max_number_current,on='key',how='left')
 best_model_key=best_model_key.sort_values('ConfirmedCases',ascending=False ).reset_index(drop=True )<groupby>
from keras.layers import ZeroPadding2D, Conv2D, MaxPooling2D, Flatten, Dense, Dropout, Input
from keras.layers import GlobalAveragePooling2D, MaxPooling2D
from keras.models import Model, Sequential
from keras.callbacks import ModelCheckpoint
from keras import regularizers
train_ck=train.groupby(['Country/Region','Date'] ).sum().reset_index()
train_ck['key']=train_ck['Country/Region']<create_dataframe>
input_layer = Input(shape=(224,224, 3))
conv = Conv2D(filters=8, kernel_size=2 )(input_layer)
 conv = Conv2D(filters=16, kernel_size=2, activation='relu' )(conv)
 conv = Conv2D(filters=32, kernel_size=2, activation='relu' )(conv)
 conv = MaxPooling2D()(conv)
 conv = Conv2D(filters=64, kernel_size=2, activation='relu' )(conv)
 conv = Conv2D(filters=128, kernel_size=2, activation='relu' )(conv)
 conv = Conv2D(filters=512, kernel_size=2, activation='relu' )(conv)
 conv = GlobalAveragePooling2D()(conv)
 dense = Dense(units=500, activation='relu' )(conv)
 dense = Dropout(0.1 )(dense)
 dense = Dense(units=100, activation='relu' )(dense)
 dense = Dropout(0.1 )(dense)
 output = Dense(units=10, activation='softmax' )(dense)
 model = Model(inputs=input_layer, outputs = output )
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train_ck['key'].unique() :
chk=train_ck[train_ck['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys )<feature_engineering>
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] ,
ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] ,
Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'] )<remove_duplicates>
checkpoint = ModelCheckpoint('best_model_1.hdf5', save_best_only=True, verbose=1)
 history = model.fit_generator(train_generator, steps_per_epoch=len(train_generator),
epochs=10,
validation_data = val_generator,
validation_steps=len(val_generator),
callbacks=[checkpoint] )
best_model_key=ConfirmedCases_all_result_Final[['key','best_model']].drop_duplicates()
max_number_current=train_ck.groupby('key' ).max() [['ConfirmedCases','Fatalities']].reset_index()
best_model_key=best_model_key.merge(max_number_current,on='key',how='left')
 best_model_key=best_model_key.sort_values('ConfirmedCases',ascending=False ).reset_index(drop=True )<set_options>
!ls.. /input/state-farm-distracted-driver-detection/
batch_index = 0
files_list = os.listdir(".. /input/state-farm-distracted-driver-detection/test/")
 def load_test_images(batch_size=32, src='.. /input/state-farm-distracted-driver-detection/test/'):
global batch_index, files_list
imgs_list = files_list[batch_index: batch_index+batch_size]
batch_index += len(imgs_list)
 batch_imgs = []
for img_name in imgs_list:
img = Image.open(src+img_name)
 im = img.resize(( 224,224))
batch_imgs.append(np.array(im)/255.)
return np.array(batch_imgs)
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<feature_engineering>
preds_list = np.array([])
 batch_index=0
batch_size = 32
while True:
tst_imgs = load_test_images(batch_size=batch_size)
 if(tst_imgs.shape[0] <= 0):
print("Batchsize is less : ",batch_index)
 break
preds = model.predict(tst_imgs)
 print("\r {}, batch_size : {}, nth_batch/all_batch : {}/{}".format(preds_list.shape,batch_size, batch_index, len(files_list)) ,end="")
 sys.stdout.flush()
if len(preds_list)== 0:
preds_list = np.array(preds)
 else:
preds_list = np.append(preds_list, preds, axis=0)
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<load_from_csv>
titles = "img,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9".split(",")
 names = pd.DataFrame(files_list[:len(preds_list)])
 names.columns=["img"]
df = pd.DataFrame(preds_list)
 df.columns=titles[1:]
df['img']=names['img']
df = df[titles]
df.tail()
df.to_csv('sub.csv',index=False)
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<merge>
def load_VGG16(weights_path=None, no_top=True):
input_shape =(224, 224, 3)
 img_input = Input(shape=input_shape)
 x = Conv2D(64,(3, 3), activation='relu', padding='same', name='block1_conv1' )(img_input)
 x = Conv2D(64,(3, 3), activation='relu', padding='same', name='block1_conv2' )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2), name='block1_pool' )(x)
 x = Conv2D(128,(3, 3), activation='relu', padding='same', name='block2_conv1' )(x)
 x = Conv2D(128,(3, 3), activation='relu', padding='same', name='block2_conv2' )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2), name='block2_pool' )(x)
 x = Conv2D(256,(3, 3), activation='relu', padding='same', name='block3_conv1' )(x)
 x = Conv2D(256,(3, 3), activation='relu', padding='same', name='block3_conv2' )(x)
 x = Conv2D(256,(3, 3), activation='relu', padding='same', name='block3_conv3' )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2), name='block3_pool' )(x)
 x = Conv2D(512,(3, 3), activation='relu', padding='same', name='block4_conv1' )(x)
 x = Conv2D(512,(3, 3), activation='relu', padding='same', name='block4_conv2' )(x)
 x = Conv2D(512,(3, 3), activation='relu', padding='same', name='block4_conv3' )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2), name='block4_pool' )(x)
 x = Conv2D(512,(3, 3), activation='relu', padding='same', name='block5_conv1' )(x)
 x = Conv2D(512,(3, 3), activation='relu', padding='same', name='block5_conv2' )(x)
 x = Conv2D(512,(3, 3), activation='relu', padding='same', name='block5_conv3' )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2), name='block5_pool' )(x)
 x = GlobalAveragePooling2D()(x)
 vmodel = Model(img_input, x, name='vgg16')
 if weights_path is not None:
print("Weights have been loaded.")
 vmodel.load_weights(weights_path)
 return vmodel
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )<categorify>
vgg_model_raw = load_VGG16('.. /input/vgg16-weights/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')
 vgg_model = vgg_model_raw.output
vgg_model = Dense(5000, activation='relu',kernel_regularizer=regularizers.l2(0.00001))(vgg_model)
 vgg_model = Dropout(0.1 )(vgg_model)
 vgg_model = Dense(500, activation='relu',kernel_regularizer=regularizers.l2(0.00001))(vgg_model)
 vgg_model = Dropout(0.1 )(vgg_model)
 vgg_model = Dense(10, activation='softmax' )(vgg_model)
 vgg_m = Model(inputs=vgg_model_raw.input, outputs= vgg_model )
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )<prepare_x_and_y>
vgg_m.layers[16].get_weights()
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x<categorify>
vgg_m.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.SGD(0.001), metrics=['accuracy'])
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )<init_hyperparams>
checkpoint = ModelCheckpoint('vgg_model.h5', save_best_only=True, verbose=1)
 history = vgg_m.fit_generator(train_generator, steps_per_epoch=len(train_generator),
epochs=6,
validation_data = val_generator,
validation_steps=len(val_generator),
callbacks=[checkpoint])
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.5
lr=1e-2
weight_decay=1e-6
batch_size=32
epochs=20
num_features=num_features
cat_features=cat_features
target_cols=target_cols<choose_model_class>
!ls
batch_index = 0
files_list = os.listdir(".. /input/state-farm-distracted-driver-detection/test")
 def load_test_images(batch_size=32, src='.. /input/state-farm-distracted-driver-detection/test/'):
global batch_index, files_list
imgs_list = files_list[batch_index: batch_index+batch_size]
batch_index += len(imgs_list)
 batch_imgs = []
for img_name in imgs_list:
img = Image.open(src+img_name)
 im = img.resize(( 224,224))
batch_imgs.append(np.array(im)/255.)
return np.array(batch_imgs)
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x<train_on_grid>
preds_list = np.array([])
 batch_index=0
batch_size = 32
mm_raw = load_VGG16()
mm_model = mm_raw.output
mm_model = Dense(5000, activation='relu',kernel_regularizer=regularizers.l2(0.00001))(mm_model)
 mm_model = Dropout(0.1 )(mm_model)
 mm_model = Dense(500, activation='relu',kernel_regularizer=regularizers.l2(0.00001))(mm_model)
 mm_model = Dropout(0.1 )(mm_model)
 mm_model = Dense(10, activation='softmax' )(mm_model)
 mm = Model(inputs=vgg_model_raw.input, outputs= vgg_model)
 mm.load_weights('vgg_model.h5')
 while True:
tst_imgs = load_test_images(batch_size=batch_size)
 if(tst_imgs.shape[0] <= 0):
print("Batchsize is less : ",batch_index)
 break
preds = mm.predict(tst_imgs)
 print("\r {}, batch_size : {}, nth_batch/all_batch : {}/{}".format(preds_list.shape,batch_size, batch_index, len(files_list)) ,end="")
 sys.stdout.flush()
if len(preds_list)== 0:
preds_list = np.array(preds)
 else:
preds_list = np.append(preds_list, preds, axis=0)
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<create_dataframe>
titles = "img,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9".split(",")
 names = pd.DataFrame(files_list[:len(preds_list)])
 names.columns=["img"]
df = pd.DataFrame(preds_list)
 df.columns=titles[1:]
df['img']=names['img']
df = df[titles]
df.tail()
<categorify><EOS> 
df.to_csv('sub_VGG16.csv',index=False )