Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
import numpy as np
import pandas as pd
import os
import torch
import torch.nn as nn
import torch.optim as optim
from PIL import Image
from matplotlib import pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms as T
from torchvision import models
import tqdm
from sklearn.metrics import f1_score,roc_auc_score,accuracy_score,confusion_matrix
class NonLinearTransformer(TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = X.drop(columns=['id'])
 for c in X.columns:
if c == 'target':
continue
X[f'{c}^2'] = X[c] ** 2
return X<define_search_model>
train_df=pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_df=pd.read_csv(".. /input/digit-recognizer/test.csv" )
pipe_xgb = Pipeline([ 
('custom', NonLinearTransformer()), 
('scaling', StandardScaler()), 
('regression', xgb.XGBRegressor(**xgb_parameters))
])
 pipe_lgb = Pipeline([ 
('custom', NonLinearTransformer()), 
('scaling', StandardScaler()), 
('regression', lgb.LGBMRegressor(**lgb_parameters))
] )<load_from_csv>
def get_image(data_df,idx):

 return Image.fromarray(np.uint8(np.reshape(data_df.iloc[idx][data_df.columns[-784:]].to_numpy() ,(28,28)))).convert('RGB')
df_train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 df_predict = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv' )<prepare_x_and_y>
class TrainDataSet(Dataset):
def __init__(self,data_df,transforms=T.ToTensor()):

 self.data_df=data_df
self.transform=transforms
def __len__(self):
return self.data_df.shape[0]
def __getitem__(self,idx):
image=self.transform(get_image(self.data_df,idx))
label=torch.tensor(self.data_df.label.iloc[idx],dtype=torch.long)
 return image,label
X, y = df_train.drop(columns=['target']), df_train['target']<split>
class TestDataSet(TrainDataSet):
def __getitem__(self,idx):
image=self.transform(get_image(self.data_df,idx))
return image
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1337 )<compute_test_metric>
def create_model() :
model = models.resnet18(pretrained=True)
 num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, 10)
 return model
pipe_xgb.fit(X_train, y_train)
 pipe_lgb.fit(X_train, y_train)
 print(f'XGB Score: {pipe_xgb.score(X_test, y_test)}, LGB Score: {pipe_lgb.score(X_test, y_test)}')
 print(f'XGB RMSE: {mean_squared_error(y_test, pipe_xgb.predict(X_test), squared=False)}, LGB RMSE: {mean_squared_error(y_test, pipe_lgb.predict(X_test), squared=False)}' )<predict_on_test>
transform=T.Compose([
T.Resize(( 256,256)) ,
T.ToTensor() ,
T.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225))
] )
def ensemble_predict(X):
target_xgb = pipe_xgb.predict(X)
 target_lgb = pipe_lgb.predict(X)
 return [0.85 * x + 0.15 * l for(x, l)in zip(target_xgb, target_lgb)]<compute_test_metric>
def train_once(model,dataloader,criterion,optimizer,device):
total_loss=0
n_total=0
criterion.reduction="sum"
model.train()
for i,(images,labels)in enumerate(tqdm.tqdm(dataloader)) :
optimizer.zero_grad()
images=images.to(device)
 labels=labels.to(device)
 outputs=model(images)
 loss=criterion(outputs,labels)
 total_loss+=loss.item()
n_total+=labels.shape[0]
loss.backward()
optimizer.step()
return total_loss/n_total
print(f'Ensemble RMSE: {mean_squared_error(y_test, ensemble_predict(X_test), squared=False)}' )<train_model>
class Validation_Metrics(object):
def __init__(self,activation_func=nn.Softmax(dim=1)) :
self.predictions=[]
self.labels=[]
self.activation_func=activation_func
self.collapsed=False
def update(self,model_outputs,labels):

 if not self.collapsed:
self.predictions.append(self.activation_func(model_outputs ).detach())
 self.labels.append(labels.detach())
 else:
raise ValueError('Error, one cannot add further values to a logger once it has been collapsed')
 def collapse(self):

 if self.collapsed:
pass
else:
self.predictions=torch.cat(self.predictions ).cpu().numpy()
self.labels=torch.cat(self.labels ).cpu().numpy()
self.collapsed=True
def Confusion_matrix(self):

 self.collapse()
Confusion_matrix=np.zeros(10,10)
 pred=np.argmax(self.predictions,axis=1)
 labels=self.labels
return confusion_matrix(labels,pred)
 def AUC(self):

 self.collapse()
pred=self.predictions
labels=np.zeros(pred.shape)
 labels[np.arange(label.shape[0]),self.labels]=1.0
aucs = []
for i in range(labels.shape[1]):
aucs.append(roc_auc_score(labels[:, i], pred[:, i]))
return aucs
def F1_score(self):
self.collapse()
pred=np.argmax(self.predictions,axis=1)
 labels=self.labels
return f1_score(labels, pred, average=None)
 def Accuracy(self):
self.collapse()
pred=np.argmax(self.predictions,axis=1)
 labels=self.labels
return accuracy_score(labels, pred)
pipe_xgb.fit(X, y)
 pipe_lgb.fit(X, y )<save_to_csv>
def val(model,dataloader,criterion,device):
total_loss=0
n_total=0
criterion.reduction="sum"
Metrics=Validation_Metrics()
model.eval()
with torch.no_grad() :
for images,labels in tqdm.tqdm(dataloader):
images=images.to(device)
 labels=labels.to(device)
 outputs=model(images)
 loss=criterion(outputs,labels)
 Metrics.update(outputs,labels)
 total_loss+=loss.item()
n_total+=labels.shape[0]
return total_loss/n_total,Metrics
target = pd.DataFrame({
'id': df_predict['id'], 'target': ensemble_predict(df_predict)
 })
 target.to_csv('submission.csv', index=False )<define_variables>
n_folds=5
PATH = '/kaggle/input/tabular-playground-series-jan-2021/'<load_from_csv>
train_df.insert(1,"fold",np.random.randint(1,n_folds+1,size=train_df.shape[0]))
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv')
 submission = pd.read_csv(PATH+'sample_submission.csv' )<install_modules>
def Get_Train_Val_Set(fold_i,transform=transform):
train_set=TrainDataSet(train_df[train_df.fold!=fold_i],transforms=transform)
 test_set=TrainDataSet(train_df[train_df.fold==fold_i],transforms=transform)
 return train_set, test_set
!pip install pycaret<import_modules>
USE_CUDA = torch.cuda.is_available()
device = torch.device("cuda" if USE_CUDA else "cpu" )
from pycaret.regression import *<define_variables>
criterion=nn.CrossEntropyLoss()
optimizer_name="Adam"
optimizer_parameters={"lr":0.0001}
epochs=1
reg = setup(data=train, target='target', silent=True, session_id=2021 )<choose_model_class>
def create_optimizer(model,optimizer_name,optimizer_parameters):
if optimizer_name=="SGD":
return optim.SGD(model.parameters() ,**optimizer_parameters)
 elif optimizer_name=="Adam":
return optim.Adam(model.parameters() ,**optimizer_parameters )
blended = blend_models(best_3, fold=5 )<predict_on_test>
Best_val_accuracy=0
for fold in range(1,n_folds+1):
print(f"Training fold {fold}")
 model=create_model()
model.to(device)
 optimizer=create_optimizer(model,optimizer_name,optimizer_parameters)
 TrainSet,ValSet=Get_Train_Val_Set(fold)
 TrainLoader=DataLoader(TrainSet, batch_size=256)
 ValLoader=DataLoader(ValSet, batch_size=1024)
 for epoch in range(epochs):
train_loss=train_once(model,TrainLoader,criterion,optimizer,device)
 print(f"For epoch {epoch+1}, the Train Loss was: {train_loss}")
 val_loss,Metrics=val(model,ValLoader,criterion,device)
 print(f"The Val Loss was {val_loss}, and the val accuracy was {Metrics.Accuracy() }")
 if Metrics.Accuracy() >Best_val_accuracy:
print("New Best, saving")
 torch.save(model.state_dict() ,f"fold{fold}Best.pt")
pred_holdout = predict_model(blended )<categorify>
optimizer=create_optimizer(model,optimizer,optimizer_parameters)
 optimizer
final_model = finalize_model(blended )<predict_on_test>
model_outputs=torch.zeros(( test_df.shape[0],10)).to(device)
 plot_every=1
predictions = predict_model(final_model, data=test )<prepare_output>
TestSet=TestDataSet(test_df,transforms=transform )
submission['target'] = predictions['Label']<save_to_csv>
for fold in range(1,n_folds+1):
print(f"Running on fold {fold}")
 model=create_model()
model.load_state_dict(torch.load(f"fold{fold}Best.pt"))
model.to(device)
 model.eval()
for i,image in enumerate(tqdm.tqdm(TestSet)) :
image=torch.unsqueeze(image,0 ).to(device)
 outputs=model(image)
 model_outputs[i]+=outputs.detach() [0]
submission.to_csv('submission_0116_baseline.csv', index=False )<import_modules>
submission_df=pd.read_csv(".. /input/digit-recognizer/sample_submission.csv" )
import lightgbm as lgb
import optuna.integration.lightgbm as oplgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
submission_df["Label"]=np.argmax(model_outputs.cpu().numpy() ,1 )
df_train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 df_test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 df_sample = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv" )<drop_column>
submission_df.to_csv("submission.csv", index=False )
<define_variables><EOS> 
pd.read_csv("submission.csv" )