Kaggle competition: State Farm Distracted Driver Detection
df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1)
 df_submit=df_submit.fillna(method='pad' ).astype(int )<merge>
! nvidia-smi
df_submit.rename(columns={0: 'ForecastId', 1: 'ConfirmedCases',2: 'Fatalities',}, inplace=True)
 complete_test= pd.merge(test, df_submit, how="left", on="ForecastId" )<save_to_csv>
! pip install torch
! pip install torchvision
df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 complete_test.to_csv('complete_test.csv',index=False)
<load_from_csv>
import pandas as pd
import subprocess
import torch
import torch.optim as optim
from torch import nn
from torch.utils.data import Dataset
from torch.utils.data.sampler import SubsetRandomSampler
import torchvision
from torchvision import transforms
import os
import random
from glob import glob
import cv2
import numpy as np
from tqdm.notebook import tqdm
train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 submission = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv')
<data_type_conversions>
EPOCH = 100
BATCH_SIZE = 16
PATIENCE = 5
DATA_PATH = '.. /input/state-farm-distracted-driver-detection'
MODEL_NAME = './model.baseline.driver_split.data_aug'
test['Date']=test.Date.astype('datetime64[ns]' )<data_type_conversions>
transform = transforms.Compose([
transforms.RandomAffine(30, translate=(0.3, 0.3)) ,
transforms.RandomPerspective(p=0.1),
transforms.RandomRotation(degrees=30),
transforms.Resize(( 224, 224)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] )
train['key']=train['Province/State'].astype('str')+ " " + train['Country/Region'].astype('str')+ " " +train['Lat'].astype('str')+ " " +train['Long'].astype('str')
 test['key']=test['Province/State'].astype('str')+ " " + test['Country/Region'].astype('str')+ " " +test['Lat'].astype('str')+ " " +test['Long'].astype('str')
 train.describe()<groupby>
classes = [f'c{i}' for i in range(10)]
seed = 2020
validation_split = 0.2
driver_list = pd.read_csv(f'{DATA_PATH}/driver_imgs_list.csv')
 drivers = np.unique(driver_list['subject'].values)
 split = int(np.floor(validation_split * len(drivers)))
 np.random.seed(seed)
 trn_idx, val_idx = drivers[split:], drivers[:split]
daily_analysis=train.groupby(['Date'] ).sum()<compute_test_metric>
train_dataset = torchvision.datasets.ImageFolder(f'./{split_dir}/train',
transform=transform)
 train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2)
 valid_transform = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 valid_dataset = torchvision.datasets.ImageFolder(f'./{split_dir}/valid',
transform=valid_transform)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=BATCH_SIZE,
num_workers=2 )
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<feature_engineering>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )
train_lag_1=train.groupby(['key'] ).shift(periods=1)
 train_lag_2=train.groupby(['key'] ).shift(periods=2)
 train_lag_3=train.groupby(['key'] ).shift(periods=3)
 train['lag_1_ConfirmedCases']=train_lag_1['ConfirmedCases']
train['lag_1_Fatalities']=train_lag_1['Fatalities']
train['lag_2_ConfirmedCases']=train_lag_2['ConfirmedCases']
train['lag_2_Fatalities']=train_lag_2['Fatalities']
train['lag_3_ConfirmedCases']=train_lag_3['ConfirmedCases']
train['lag_3_Fatalities']=train_lag_3['Fatalities']
train<create_dataframe>
model_conv = torchvision.models.resnet50(pretrained=True)
 num_ftrs = model_conv.fc.in_features
model_conv.fc = nn.Sequential(
 nn.Linear(num_ftrs, num_ftrs),
nn.ReLU() ,
nn.Dropout(0.5),
nn.Linear(num_ftrs, num_ftrs),
nn.ReLU() ,
nn.Dropout(0.5),
nn.Linear(num_ftrs, len(classes)))
 print(f'
model_conv = model_conv.to(device )
orig_stdout = sys.stdout
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train['key'].unique() :
chk=train[train['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys)
 sys.stdout = orig_stdout<feature_engineering>
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_conv.parameters() , lr=0.0001, weight_decay=1e-6, momentum=0.9)
 softmax = nn.Softmax(dim=1)
 best_valid_score = 999
patience = 0
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] ,
ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] ,
Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'] )<data_type_conversions>
def train(model_conv, train_loader, optimizer, criterion, trn_cnt):
running_loss = 0.
 running_acc = 0.
 pbar = tqdm(total=trn_cnt)
 cnt = 0
for i, data in enumerate(train_loader, 0):
inputs, labels = data
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
model_conv.train()
outputs = model_conv(inputs)
 probs = softmax(outputs)
 _, preds = probs.max(axis=1)
 running_acc += sum(labels == preds)/(1.* BATCH_SIZE)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
cnt += 1
pbar.update(BATCH_SIZE)
 pbar.close()
return running_loss / cnt, running_acc / cnt
test['Date']=test.Date.astype('datetime64[ns]' )<merge>
def evaluate(model_conv, valid_loader, criterion):
with torch.no_grad() :
model_conv.eval()
valid_loss = 0.0
valid_acc = 0.0
cnt = 0
pbar = tqdm(total=val_cnt)
 for data in valid_loader:
inputs, labels = data
inputs = inputs.to(device)
 labels = labels.to(device)
 outputs = model_conv(inputs)
 probs = softmax(outputs)
 _, preds = probs.max(axis=1)
 valid_acc += sum(labels == preds)/(1.* BATCH_SIZE)
 labels.require_grad = False
loss = criterion(outputs, labels)
 valid_loss += loss
cnt += 1
pbar.update(BATCH_SIZE)
 pbar.close()
return valid_loss / cnt, valid_acc / cnt
eval1 = ConfirmedCases_all_result_Final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval1.rename(columns={'best_pred':'ConfirmedCases'},inplace=True)
 eval1['ConfirmedCases']=eval1['ConfirmedCases'].fillna(0)
 eval1<merge>
for epoch in range(EPOCH):
print(f'
trn_loss, trn_acc = train(model_conv, train_loader, optimizer, criterion, trn_cnt)
 print(f'
valid_loss, valid_acc = evaluate(model_conv, valid_loader, criterion)
 print(f'
if valid_loss < best_valid_score:
best_valid_score = valid_loss
print(f'
torch.save(model_conv, MODEL_NAME)
 patience = 0
patience += 1
if patience == PATIENCE:
break
print('Finished Training' )
<merge><EOS> 
TEST_SIZE = 79726
BATCH_SIZE = 128
test_ids = [os.path.basename(fl)for fl in glob(f'{DATA_PATH}/imgs/test/img_*.jpg')]
test_ids.sort()
transform = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 test_dataset = torchvision.datasets.ImageFolder(f'{DATA_PATH}/imgs',
transform=transform)
 test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=BATCH_SIZE,
num_workers=2)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 model_conv = torch.load(MODEL_NAME)
 print(f'
model_conv = model_conv.to(device)
 softmax = nn.Softmax(dim=1)
 pbar = tqdm(total=TEST_SIZE)
 end_flag = False
with open('submission.csv', 'w')as out:
out.write('img,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9
')
 for i, data in enumerate(test_loader, 0):
inputs, labels = data
inputs = inputs.to(device)
 labels = labels.to(device)
 with torch.no_grad() :
model_conv.eval()
outputs = model_conv(inputs)
 probs = softmax(outputs)
 for j, prob in enumerate(probs):
if BATCH_SIZE * i + j >= TEST_SIZE:
end_flag = True
break
test_id = test_ids[i * BATCH_SIZE + j]
prob = ','.join([str(round(val, 3)) for val in prob.cpu().detach().numpy() ])
 out.write(f'{test_id},{prob}
')
 pbar.update(BATCH_SIZE)
 if end_flag:
break
pbar.close()
print('Finished Eval' )