Kaggle competition: PadhAI: Hindi Vowel - Consonant Classification
<SOS> metric: categorizationaccuracy Kaggle data source: padhai:-hindi-vowel-consonant-classification<feature_engineering>
class VowelConsonantDataset(Dataset):
def __init__(self, file_path,train=True,transform=None):
self.transform = transform
self.file_path=file_path
self.train=train
self.file_names=[file for _,_,files in os.walk(self.file_path)for file in files]
self.len = len(self.file_names)
 if self.train:
self.classes_mapping=self.get_classes()
def __len__(self):
return len(self.file_names)
 def __getitem__(self, index):
file_name=self.file_names[index]
image_data=self.pil_loader(self.file_path+"/"+file_name)
 if self.transform:
image_data = self.transform(image_data)
 if self.train:
file_name_splitted=file_name.split("_")
 Y1 = self.classes_mapping[file_name_splitted[0]]
Y2 = self.classes_mapping[file_name_splitted[1]]
z1,z2=torch.zeros(10),torch.zeros(10)
 z1[Y1-10],z2[Y2]=1,1
label=torch.stack([z1,z2])
 return image_data, label
else:
return image_data, file_name
def pil_loader(self,path):
with open(path, 'rb')as f:
img = Image.open(f)
 return img.convert('RGB')
 def get_classes(self):
classes=[]
for name in self.file_names:
name_splitted=name.split("_")
 classes.extend([name_splitted[0],name_splitted[1]])
 classes=list(set(classes))
classes_mapping={}
for i,cl in enumerate(sorted(classes)) :
classes_mapping[cl]=i
return classes_mapping
train['abs_diff_longitude'] = np.abs(train['dropoff_longitude'] - train['pickup_longitude'])
 train['abs_diff_latitude'] = np.abs(train['dropoff_latitude'] - train['pickup_latitude'])
 test['abs_diff_longitude'] = np.abs(test['dropoff_longitude'] - test['pickup_longitude'])
 test['abs_diff_latitude'] = np.abs(test['dropoff_latitude'] - test['pickup_latitude'] )<filter>
train_on_gpu = torch.cuda.is_available()
train = train.loc[train['fare_amount']>0,:]
train = train.loc[(train["passenger_count"]<=6)&(train["passenger_count"]>0),:]
train = train.loc[(train["abs_diff_latitude"]<2)&(train["abs_diff_longitude"]<2),:]<filter>
transform_train = transforms.Compose([
transforms.ColorJitter() ,
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor() ,
] )
train = train[train['fare_amount'].between(left = 2.5, right = 100)]<data_type_conversions>
full_data = VowelConsonantDataset(".. /input/train/train",train=True,transform=transform_train)
 train_size = int(0.9 * len(full_data))
test_size = len(full_data)- train_size
train_data, validation_data = random_split(full_data, [train_size, test_size])
 train_loader = torch.utils.data.DataLoader(train_data, batch_size=50, shuffle=True)
 validation_loader = torch.utils.data.DataLoader(validation_data, batch_size=50, shuffle=True )
train.loc[:,'timestamp_with_key'] = train.loc[:,'key']
test.loc[:,'timestamp_with_key'] = test.loc[:,'key']
train.key = pd.DataFrame({'key':train['key'].str.split('.' ).str[1].astype('int')})
 test.key = pd.DataFrame({'key':test['key'].str.split('.' ).str[1].astype('int')} )<data_type_conversions>
test_data = VowelConsonantDataset(".. /input/test/test",train=False,transform=transform_train)
 test_loader = torch.utils.data.DataLoader(test_data, batch_size=20,shuffle=False )
<data_type_conversions>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'], infer_datetime_format=True ).dt.tz_localize('UTC')
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'], infer_datetime_format=True ).dt.tz_localize('UTC')
 train.loc[:,'hour_no'] = train['pickup_datetime'][:].dt.strftime('%-H' ).astype('int')
 test.loc[:,'hour_no'] = test['pickup_datetime'][:].dt.strftime('%-H' ).astype('int')
 train.loc[:,'weekday_no'] = train['pickup_datetime'][:].dt.strftime('%w' ).astype('int')
 test.loc[:,'weekday_no'] = test['pickup_datetime'][:].dt.strftime('%w' ).astype('int')
 train.loc[:,'day_no'] = train['pickup_datetime'][:].dt.strftime('%-d' ).astype('int')
 test.loc[:,'day_no'] = test['pickup_datetime'][:].dt.strftime('%-d' ).astype('int')
 train.loc[:,'year_no'] = train['pickup_datetime'][:].dt.strftime('%-y' ).astype('int')
 test.loc[:,'year_no'] = test['pickup_datetime'][:].dt.strftime('%-y' ).astype('int' )<create_dataframe>
class MyModel(nn.Module):
def __init__(self, num_classes1, num_classes2):
super(MyModel, self ).__init__()
self.model_snet = models.mobilenet_v2(pretrained=True)
 self.model_snet.classifier = nn.Sequential(nn.Dropout(p=0.2))
self.fc1 = nn.Linear(1280, num_classes1,bias=True)
 torch.nn.init.xavier_uniform_(self.fc1.weight)
 torch.nn.init.zeros_(self.fc1.bias)
 self.fc2 = nn.Linear(1280, num_classes2,bias=True)
 torch.nn.init.xavier_uniform_(self.fc2.weight)
 torch.nn.init.zeros_(self.fc2.bias)
 def forward(self, x):
x = self.model_snet(x)
 out1 = self.fc1(x)
 out2 = self.fc2(x)
 return out1, out2
<compute_test_metric>
my_model = MyModel(10,10 )
def dist_haversine(x):
R = 6371
picklat = math.radians(x[1])
 droplat = math.radians(x[3])
 latdiff = abs(droplat-picklat)
 picklon = math.radians(x[0])
 droplon = math.radians(x[2])
 londiff = abs(droplon-picklon)
 a = math.sin(latdiff/2)* math.sin(latdiff/2)+\
math.cos(picklat)* math.cos(droplat)*\
math.sin(londiff/2)* math.sin(londiff/2)
 c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
return(R * c)
 train['dist_haversine_km'] = pd.DataFrame(list(
 map(lambda x: dist_haversine(x), train[["pickup_longitude","pickup_latitude","dropoff_longitude","dropoff_latitude"]].values)) ,
index=train.index)
 test['dist_haversine_km'] = pd.DataFrame(list(
 map(lambda x: dist_haversine(x), test[["pickup_longitude","pickup_latitude","dropoff_longitude","dropoff_latitude"]].values)) ,
index=test.index)
<prepare_x_and_y>
my_model = my_model.to(device )
<feature_engineering>
def evaluation(dataloader,model):
total,correct=0,0
for data in dataloader:
inputs,labels=data
inputs,labels=inputs.to(device),labels.to(device)
 out1,out2=my_model(inputs)
 _,pred1=torch.max(out1.data,1)
 _,pred2=torch.max(out2.data,1)
 _,labels1=torch.max(labels[:,0,:].data,1)
 _,labels2=torch.max(labels[:,1,:].data,1)
 total+=labels.size(0)
 fin1=(pred1==labels1)
 fin2=(pred2==labels2)
 correct+=(fin1==fin2 ).sum().item()
return 100*correct/total
train['fare_per_km'] = train['fare_amount']/(train["dist_haversine_km"])
 train['fare_per_km_passenger'] = train['fare_amount']/(train['dist_haversine_km']*train['passenger_count'] )<groupby>
loss_fn = nn.CrossEntropyLoss()
plist = [
{'params': my_model.fc1.parameters() , 'lr': 5e-3},
{'params': my_model.fc2.parameters() , 'lr': 5e-3}
]
lr=0.01
opt = optim.SGD(my_model.parameters() ,lr=0.01,momentum=0.9,nesterov=True)
train.groupby('key' ).agg({'fare_per_km_passenger':'mean','key':'count','passenger_count':'mean','fare_amount':'mean'} )<filter>
loss_epoch_arr = []
loss_arr = []
max_epochs = 10
min_loss = 1000
batch_size = 50
n_iters = np.ceil(9000/batch_size)
 for epoch in range(max_epochs):
for i, data in enumerate(train_loader, 0):
my_model.train()
images, labels = data
images = images.to(device)
 targetnp=labels[:,0,:].cpu().numpy()
targetnp1 = labels[:,1,:].cpu().numpy()
new_targets1 = np.argmax(targetnp,axis=1)
 new_targets2 = np.argmax(targetnp1,axis=1)
 new_targets1=torch.LongTensor(new_targets1)
 new_targets2=torch.LongTensor(new_targets2)
 new_targets1 = new_targets1.to(device)
 new_targets2 = new_targets2.to(device)
 opt.zero_grad()
out = my_model.forward(images)
 loss_fc1 = loss_fn(out[0], new_targets1)
 loss_fc2 = loss_fn(out[1],new_targets2)
 loss = torch.add(loss_fc1,loss_fc2)
 loss.backward()
opt.step()
if min_loss > loss.item() :
min_loss = loss.item()
best_model = copy.deepcopy(my_model.state_dict())
 print('Min loss %0.2f' % min_loss)
 if i % 100 == 0:
print('Iteration: %d/%d, Loss: %0.2f' %(i, n_iters, loss.item()))
del images, labels, out
torch.cuda.empty_cache()
loss_arr.append(loss.item())
 print("Epoch number :",epoch)
 print("Train Accuracy :",evaluation(train_loader,my_model))
print("Test Accuracy :" ,evaluation(validation_loader,my_model))
print('Learning Rate :',lr)
 loss_epoch_arr.append(loss.item())
 plt.plot(loss_arr)
 plt.show()
train.loc[train['fare_per_km_passenger']>20,['fare_per_km_passenger','fare_amount','dist_haversine_km']] 
<drop_column>
evaluation(validation_loader,my_model )
del reindexed, grouped, get_max_count, grouped_df<filter>
my_model.eval()
plist=[]
fn_list=[]
for inputs_test, fn in test_loader:
inputs_test=inputs_test.to(device)
 out1,out2=my_model.forward(inputs_test)
 _,pred1=torch.max(out1,1)
 pred1=pred1.tolist()
_,pred2=torch.max(out2,1)
 pred2=pred2.tolist()
for x,y,z in zip(pred1,pred2,fn):
p="V"+str(x)+"_"+"C"+str(y)
 plist.append(p)
 fn_list.append(z )
train = train.loc[ ~(((train['dist_haversine_km']>1)& train['fare_per_km']<0.2)) ]
train = train.loc[~(( train['dist_haversine_km']<0.01)&(train['fare_per_km']>50)) ]<drop_column>
submission = pd.DataFrame({"ImageId":fn_list, "Class":plist})
 submission.to_csv('submission.csv', index=False )