Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<count_values>
!pip install torchsummary
print('Number of genres in films:')
 train['genres'].apply(lambda x: len(x)if x!={} else 0 ).value_counts()<define_variables>
df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test_df = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 print(df.shape, test_df.shape)
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x!={} else [] ).values )<feature_engineering>
class MNIST_dataset(Dataset):
def __init__(self, df):
self.df = df
self.aug = A.Compose([
A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=10, p=.75)
 ])
 def __len__(self):
return(len(self.df))
def __getitem__(self, idx):
img_data = self.df.iloc[idx,1:].values.reshape(( 1,28,28)).astype(np.float32)/ 255.
 img_data = self.aug(image=img_data)['image']
label = self.df.iloc[idx, 0]
return img_data, label
train['num_of_genres'] = train['genres'].apply(lambda x: len(x)if x!={} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x ]))
if x!= {} else '')
 test['num_of_genres'] = test['genres'].apply(lambda x: len(x)if x!={} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x ]))
if x!= {} else '' )<feature_engineering>
train_df , valid_df = train_test_split(df, test_size=0.2, random_state=1 )
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0 )<define_variables>
train_dl = DataLoader(MNIST_dataset(train_df), batch_size=128)
 valid_dl = DataLoader(MNIST_dataset(valid_df), batch_size=128 )
for i,e in enumerate(train['production_companies'][:2]):
print(i,e )<count_values>
num_groups = 4
class Model(nn.Module):
def __init__(self):
super().__init__()
self.features = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(32),
nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(32),
nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(64),
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(128),
nn.Conv2d(128, 128, kernel_size=3, stride=2, padding=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(128),
nn.Dropout2d(0.25), 
)
 self.classifier = nn.Sequential(
 nn.Linear(4*4*128, 256),
nn.ReLU(inplace=True),
nn.BatchNorm1d(256),
nn.Dropout2d(0.25),
nn.Linear(256, 10), 
)
 def forward(self, x):
x = self.features(x)
 x = x.view(x.size(0), -1)
 x = self.classifier(x)
 return(x)
print('Number of Production Companies for a movie:')
 train['production_companies'].apply(lambda x: len(x)if x!= {} else 0 ).value_counts()<filter>
summary(model, input_size=(1, 28, 28))
train[train['production_companies'].apply(lambda x: len(x)if x!= {} else 0)> 10]<count_unique_values>
from torch import optim
from tqdm.auto import tqdm
list_of_companies = list(train['production_companies'].apply(lambda x : [i['name'] for i in x]
if x!= {} else [] ).values)
 Counter(i for j in list_of_companies for i in j ).most_common(20 )<feature_engineering>
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters() , lr=1e-3 )
train['num_prod_companies'] = train['production_companies'].apply(lambda x: len(x)if
x!={} else 0)
 test['num_prod_companies'] = test['production_companies'].apply(lambda x: len(x)if
x!={} else 0)
 train['all_prod_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x!={} else '')
 test['all_prod_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x!={} else '' )<feature_engineering>
def train_one_epoch(dl, epoch_num):
total_loss = 0.0
accumulation_steps = 1024 // 128
optimizer.zero_grad()
for i,(X, y)in enumerate(tqdm(dl)) :
y1 = model(X.cuda())
 loss = criterion(y1, y.cuda())
 loss /= accumulation_steps
loss.backward()
if(( i+1)% accumulation_steps == 0):
optimizer.step()
optimizer.zero_grad()
total_loss += loss.detach().cpu().item()
print(f'epoch : {epoch_num}, Loss : {total_loss/len(dl.dataset):.6f}')
top_prod_companies = [m[0] for m in Counter(i for j in list_of_companies for i in j ).most_common(10)]
for pc in top_prod_companies:
train['production_' + pc] = train['all_prod_companies'].apply(lambda x: 1 if pc in x else 0)
 test['production_'+ pc] = test['all_prod_companies'].apply(lambda x: 1 if pc in x else 0 )<define_variables>
def evaluate(dl):
total_loss = 0.0
total_correct = 0.0
with torch.no_grad() :
for X,y in dl :
y1 = model(X.cuda())
 loss = criterion(y1,y.cuda())
 pred = torch.argmax(y1, dim=1 ).cpu()
total_loss+=loss.item()
total_correct += torch.sum(y==pred ).float().item()
accuracy = total_correct/len(dl.dataset)
 print(f'Loss : {total_loss/len(dl.dataset):.6f}, Accuracy : {accuracy*100:.3f}%')
for i, e in enumerate(train['production_countries'][:2]):
print(i,e )<count_values>
epoch_num = 20
for epoch in range(epoch_num):
train_one_epoch(train_dl, epoch)
 evaluate(valid_dl )
print('Number of Production Countries in Movies:')
 train['production_countries'].apply(lambda x: len(x)if x!={} else 0 ).value_counts()<filter>
class MNIST_dataset(Dataset):
def __init__(self, df):
self.df = df
def __len__(self):
return(len(self.df))
def __getitem__(self, idx):
img_data = self.df.iloc[idx].values.reshape(( 1,28,28)).astype(np.float32)/ 255.
 return img_data
train[train['production_countries'].apply(lambda x: len(x)if x!= {} else 0)> 5]<count_values>
test_dl = DataLoader(MNIST_dataset(test_df), batch_size=128 )
List_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x]
if x!= {} else []))
Counter(i for j in List_of_countries for i in j ).most_common(10 )<feature_engineering>
def evaluate_Submssions(dl):
total_loss = 0.0
total_correct = 0.0
pred_list =[]
with torch.no_grad() :
for X in dl :
y1 = model(X.cuda())
 pred = torch.argmax(y1, dim=1 ).detach().cpu().numpy().tolist()
pred_list.extend(pred)
 return pred_list
train['num_prod_countries'] = train['production_countries'].apply(lambda x: len(x)if x!= {}
else 0)
 test['num_prod_countries'] = test['production_countries'].apply(lambda x: len(x)if x!={}
else 0)
 train['all_prod_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted(i['name'] for i in x))
if x!= {} else '')
 test['all_prod_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted(i['name'] for i in x))
if x!= {} else '')
<feature_engineering>
pred_list = evaluate_Submssions(test_dl)
top_prod_countries = [m[0] for m in Counter(i for j in List_of_countries for i in j ).most_common(6)]
for t in top_prod_countries:
train['prod_country_' + t] = train['all_prod_countries'].apply(lambda x: 1 if t in x else 0)
 test['prod_country_'+ t] = test['all_prod_countries'].apply(lambda x: 1 if t in x else 0 )<define_variables>
subs = pd.DataFrame({
'ImageId': range(1, len(pred_list)+1),
'Label' : pred_list
})
 subs.head()
for i, e in enumerate(train['spoken_languages'][:2]):
print(i,e )<count_values>
subs.to_csv("submission.csv", index= False )