Kaggle competition: PadhAI: Hindi Vowel - Consonant Classification
train = train.drop(['fare_per_km_passenger','fare_per_km'],axis=1 )<categorify>
class VowelConsonantDataset(Dataset):
def __init__(self, file_path,train=True,transform=None):
self.transform = transform
self.file_path=file_path
self.train=train
self.file_names=[file for _,_,files in os.walk(self.file_path)for file in files]
self.len = len(self.file_names)
 if self.train:
self.classes_mapping=self.get_classes()
def __len__(self):
return len(self.file_names)
 def __getitem__(self, index):
file_name=self.file_names[index]
image_data=self.pil_loader(self.file_path+"/"+file_name)
 if self.transform:
image_data = self.transform(image_data)
 if self.train:
file_name_splitted=file_name.split("_")
 Y1 = self.classes_mapping[file_name_splitted[0]]
Y2 = self.classes_mapping[file_name_splitted[1]]
z1,z2=torch.zeros(10),torch.zeros(10)
 z1[Y1-10],z2[Y2]=1,1
label=torch.stack([z1,z2])
 return image_data, label
else:
return image_data, file_name
def pil_loader(self,path):
with open(path, 'rb')as f:
img = Image.open(f)
 return img.convert('RGB')
 def get_classes(self):
classes=[]
for name in self.file_names:
name_splitted=name.split("_")
 classes.extend([name_splitted[0],name_splitted[1]])
 classes=list(set(classes))
classes_mapping={}
for i,cl in enumerate(sorted(classes)) :
classes_mapping[cl]=i
return classes_mapping
train = train.loc[train['pickup_latitude'].between(40, 42)]
train = train.loc[train['pickup_longitude'].between(-75, -72)]
train = train.loc[train['dropoff_latitude'].between(40, 42)]
train = train.loc[train['dropoff_longitude'].between(-75, -72)] 
<feature_engineering>
train_on_gpu = torch.cuda.is_available()
train.loc[:,'pickuplat_no'], pick_lat_bin = pd.cut(train['pickup_latitude'],100, labels=False, retbins=True)
 train.loc[:,'pickuplong_no'], pick_long_bin = pd.cut(train['pickup_longitude'],100, labels=False, retbins=True)
 train.loc[:,'dropofflat_no'], drop_lat_bin = pd.cut(train['dropoff_latitude'],100, labels=False, retbins=True)
 train.loc[:,'dropofflong_no'], drop_long_bin = pd.cut(train['dropoff_longitude'],100, labels=False, retbins=True)
 test['pickuplat_no'] = pd.cut(test['pickup_latitude'], pick_lat_bin, labels=False ).fillna(int(train.loc[:,'pickuplat_no'].mean()))
test['pickuplong_no'] = pd.cut(test['pickup_longitude'], pick_long_bin, labels=False ).fillna(int(train.loc[:,'pickuplong_no'].mean()))
test['dropofflat_no'] = pd.cut(test['dropoff_latitude'], drop_lat_bin, labels=False ).fillna(int(train.loc[:,'dropofflat_no'].mean()))
test['dropofflong_no'] = pd.cut(test['dropoff_longitude'], drop_long_bin, labels=False ).fillna(int(train.loc[:,'dropofflong_no'].mean()))<data_type_conversions>
transform_train = transforms.Compose([
transforms.ColorJitter() ,
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor() ,
] )
train.loc[:,'pickdrop_lat_diff'] = abs(train['pickuplat_no'].astype(int)- train['dropofflat_no'].astype(int))
train.loc[:,'pickdrop_long_diff'] = abs(train['pickuplong_no'].astype(int)- train['dropofflong_no'].astype(int))
train.loc[:,'final_dist_factor'] =(train['pickdrop_lat_diff'].astype(int)+ train['pickdrop_long_diff'].astype(int))
test.loc[:,'pickdrop_lat_diff'] = abs(test['pickuplat_no'].astype(int)- test['dropofflat_no'].astype(int))
test.loc[:,'pickdrop_long_diff'] = abs(test['pickuplong_no'].astype(int)- test['dropofflong_no'].astype(int))
test.loc[:,'final_dist_factor'] =(test['pickdrop_lat_diff'].astype(int)+ test['pickdrop_long_diff'].astype(int))<sort_values>
full_data = VowelConsonantDataset(".. /input/padhai-hindivowelconsonantdataset/train/train",train=True,transform=transform_train)
 train_size = int(0.9 * len(full_data))
test_size = len(full_data)- train_size
train_data, validation_data = random_split(full_data, [train_size, test_size])
 train_loader = torch.utils.data.DataLoader(train_data, batch_size=50, shuffle=True)
 validation_loader = torch.utils.data.DataLoader(validation_data, batch_size=50, shuffle=True)
<feature_engineering>
test_data = VowelConsonantDataset(".. /input/padhai-hindivowelconsonantdataset/test/test",train=False,transform=transform_train)
 test_loader = torch.utils.data.DataLoader(test_data, batch_size=20,shuffle=False)
 print(len(test_data))
def calc_cwd_factor(df, col):
new_df = df.groupby(col)['key'].count().sort_values(ascending=False ).reset_index()
new_df['cwd_factor'] = 1
count = 1
for i in range(1,new_df.shape[0]):
count += 1
if new_df.loc[i-1,'key'] == new_df.loc[i,'key']:
count -= 1
new_df.loc[i,'cwd_factor'] = count
new_df.index = new_df[col]
return new_df
fact_df = calc_cwd_factor(train, 'pickuplat_no')
 train.loc[:,'pickuplat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['pickuplat_no']))
fact_df = calc_cwd_factor(train, 'pickuplong_no')
 train.loc[:,'pickuplong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['pickuplong_no']))
fact_df = calc_cwd_factor(train, 'dropofflat_no')
 train.loc[:,'dropofflat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['dropofflat_no']))
fact_df = calc_cwd_factor(train, 'dropofflong_no')
 train.loc[:,'dropofflong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['dropofflong_no']))
fact_df = calc_cwd_factor(test, 'pickuplat_no')
 test.loc[:,'pickuplat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['pickuplat_no']))
fact_df = calc_cwd_factor(test, 'pickuplong_no')
 test.loc[:,'pickuplong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['pickuplong_no']))
fact_df = calc_cwd_factor(test, 'dropofflat_no')
 test.loc[:,'dropofflat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['dropofflat_no']))
fact_df = calc_cwd_factor(test, 'dropofflong_no')
 test.loc[:,'dropofflong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['dropofflong_no']))
del fact_df<set_options>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )
gc.collect()<import_modules>
class MyModel(nn.Module):
def __init__(self, num_classes1, num_classes2):
super(MyModel, self ).__init__()
self.model_snet = models.mobilenet_v2(pretrained=True)
 self.model_snet.classifier = nn.Sequential(nn.Dropout(p=0.2))
self.fc1 = nn.Linear(1280, num_classes1,bias=True)
 torch.nn.init.xavier_uniform_(self.fc1.weight)
 torch.nn.init.zeros_(self.fc1.bias)
 self.fc2 = nn.Linear(1280, num_classes2,bias=True)
 torch.nn.init.xavier_uniform_(self.fc2.weight)
 torch.nn.init.zeros_(self.fc2.bias)
 def forward(self, x):
x = self.model_snet(x)
 out1 = self.fc1(x)
 out2 = self.fc2(x)
 return out1, out2
import sklearn
from sklearn import *
from sklearn.preprocessing import Normalizer
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle<normalization>
my_model = MyModel(10,10 )
train = shuffle(train.iloc[:,:] ).reset_index(drop=True)
 val = train.iloc[int(0.9*train.shape[0]):,:]
train = train.iloc[:int(0.9*train.shape[0]),:] 
<define_variables>
my_model = my_model.to(device )
object_cols = ['key','pickup_datetime','timestamp_with_key']
categorical_cols = ['passenger_count', 'hour_no', 'weekday_no', 'day_no', 'year_no', 'pickuplat_no', 'pickuplong_no',
'dropofflat_no', 'dropofflong_no', 'pickdrop_lat_diff', 'pickdrop_long_diff', 'final_dist_factor',
'pickuplat_cwd_factor', 'pickuplong_cwd_factor', 'dropofflat_cwd_factor', 'dropofflong_cwd_factor']
numerical_cols =['pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude',
'abs_diff_longitude', 'abs_diff_latitude', 'dist_haversine_km']
target = ['fare_amount']
cor = train.loc[:,numerical_cols+target]
f, ax = plt.subplots(1,1,figsize=(18,7))
sns.heatmap(cor.corr() , annot=True )<set_options>
def evaluation(dataloader,model):
total,correct=0,0
for data in dataloader:
inputs,labels=data
inputs,labels=inputs.to(device),labels.to(device)
 out1,out2=my_model(inputs)
 _,pred1=torch.max(out1.data,1)
 _,pred2=torch.max(out2.data,1)
 _,labels1=torch.max(labels[:,0,:].data,1)
 _,labels2=torch.max(labels[:,1,:].data,1)
 total+=labels.size(0)
 fin1=(pred1==labels1)
 fin2=(pred2==labels2)
 correct+=(fin1==fin2 ).sum().item()
return 100*correct/total
gc.collect()<define_search_model>
loss_fn = nn.CrossEntropyLoss()
plist = [
{'params': my_model.fc1.parameters() , 'lr': 5e-3},
{'params': my_model.fc2.parameters() , 'lr': 5e-3}
]
lr=0.01
opt = optim.SGD(my_model.parameters() ,lr=0.01,momentum=0.9,nesterov=True)
f_reg = f_regression(train[categorical_cols+numerical_cols],train['fare_amount'], center=True)
 mir_c = mutual_info_regression(train[categorical_cols][:10000], train['fare_amount'][:10000])
 mir_n = mutual_info_regression(train[numerical_cols][:10000], train['fare_amount'][:10000])
 impt_cols_freg = list(zip(f_reg[0], categorical_cols+numerical_cols))
impt_cols_freg = sorted(impt_cols_freg, key = lambda x:x[0], reverse=True)
 impt_cols_c = list(zip(mir_c ,categorical_cols))
impt_cols_c = sorted(impt_cols_c, key = lambda x:x[0], reverse=True)
 impt_cols_n = list(zip(mir_n ,numerical_cols))
impt_cols_n = sorted(impt_cols_n, key = lambda x:x[0], reverse=True )<drop_column>
loss_epoch_arr = []
loss_arr = []
max_epochs = 2
min_loss = 1000
batch_size = 50
n_iters = np.ceil(9000/batch_size)
 for epoch in range(max_epochs):
for i, data in enumerate(train_loader, 0):
my_model.train()
images, labels = data
images = images.to(device)
 targetnp=labels[:,0,:].cpu().numpy()
targetnp1 = labels[:,1,:].cpu().numpy()
new_targets1 = np.argmax(targetnp,axis=1)
 new_targets2 = np.argmax(targetnp1,axis=1)
 new_targets1=torch.LongTensor(new_targets1)
 new_targets2=torch.LongTensor(new_targets2)
 new_targets1 = new_targets1.to(device)
 new_targets2 = new_targets2.to(device)
 opt.zero_grad()
out = my_model.forward(images)
 loss_fc1 = loss_fn(out[0], new_targets1)
 loss_fc2 = loss_fn(out[1],new_targets2)
 loss = torch.add(loss_fc1,loss_fc2)
 loss.backward()
opt.step()
if min_loss > loss.item() :
min_loss = loss.item()
best_model = copy.deepcopy(my_model.state_dict())
 print('Min loss %0.2f' % min_loss)
 if i % 100 == 0:
print('Iteration: %d/%d, Loss: %0.2f' %(i, n_iters, loss.item()))
del images, labels, out
torch.cuda.empty_cache()
loss_arr.append(loss.item())
 print("Epoch number :",epoch)
 print("Train Accuracy :",evaluation(train_loader,my_model))
print("Test Accuracy :" ,evaluation(validation_loader,my_model))
print('Learning Rate :',lr)
 loss_epoch_arr.append(loss.item())
 plt.plot(loss_arr)
 plt.show()
drop_cols = []
for i in impt_cols_c[-8:]:
drop_cols.append(i[1])
 train = train.drop(drop_cols, axis=1)
 test = test.drop(drop_cols, axis=1 )<set_options>
my_model.load_state_dict(best_model )
gc.collect()<prepare_x_and_y>
evaluation(validation_loader,my_model )
train_y = train['fare_amount'][:]
val_y = val['fare_amount'][:]
cols = [i for i in train.columns if i not in ['fare_amount'] + object_cols]
train_x = train.loc[:, cols]
val_x = val.loc[:, cols]
test_x = test.loc[:,cols] 
<drop_column>
my_model.eval()
plist=[]
fn_list=[]
for inputs_test, fn in test_loader:
inputs_test=inputs_test.to(device)
 out1,out2=my_model.forward(inputs_test)
 _,pred1=torch.max(out1,1)
 pred1=pred1.tolist()
_,pred2=torch.max(out2,1)
 pred2=pred2.tolist()
for x,y,z in zip(pred1,pred2,fn):
p="V"+str(x)+"_"+"C"+str(y)
 plist.append(p)
 fn_list.append(z )
<predict_on_test><EOS> 
submission = pd.DataFrame({"ImageId":fn_list, "Class":plist})
 submission.to_csv('submission.csv', index=False )