Kaggle competition: Kannada MNIST
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.5
lr=1e-2
weight_decay=1e-6
batch_size=32
epochs=20
num_features=num_features
cat_features=cat_features
target_cols=target_cols<choose_model_class>
%matplotlib inline
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x<train_on_grid>
lr_schedule = LearningRateScheduler(lambda epoch: 1e-6 * 2**(epoch/2))
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<create_dataframe>
train_path = Path.cwd() /'.. '/'input'/'Kannada-MNIST'/'train.csv'
all_data = pd.read_csv(train_path )
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr, weight_decay=cfg.weight_decay)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions<categorify>
y = all_data.label
x = all_data.iloc[:,1:].values
x = x.reshape(x.shape[0], 28, 28, 1 )
oof = np.zeros(( len(train), len(CFG.target_cols)))
 predictions = np.zeros(( len(test), len(CFG.target_cols)))
 SEED = [0, 1, 2]
for seed in SEED:
_oof, _predictions = run_kfold_nn(CFG,
train, test, folds,
num_features, cat_features, target,
device,
n_fold=5, seed=seed)
 oof += _oof / len(SEED)
 predictions += _predictions / len(SEED)
 score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"Seed Averaged CV score: {score}" )<save_to_csv>
train_datagen = ImageDataGenerator(rescale=1./255.,
rotation_range=10,
width_shift_range=0.25,
height_shift_range=0.25,
shear_range=0.1,
zoom_range=0.25,
horizontal_flip=False)
 train_generator = train_datagen.flow(x, y, batch_size=1024 )
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof.csv', index=False)
 test[target_cols] = predictions
test[['sig_id']+target_cols].to_csv('pred.csv', index=False )<compute_test_metric>
class LR_Updater(Callback):

 def __init__(self, epoch_iterations):

 self.epoch_iterations = epoch_iterations
self.trn_iterations = 0.
 self.history = {}
def on_train_begin(self, logs={}):
self.trn_iterations = 0.
 logs = logs or {}
def on_batch_end(self, batch, logs=None):
logs = logs or {}
self.trn_iterations += 1
K.set_value(self.model.optimizer.lr, self.setRate())
 self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 class LR_Cycle(LR_Updater):

 def __init__(self, iterations, cycle_mult = 1):

 self.min_lr = 0
self.cycle_mult = cycle_mult
self.cycle_iterations = 0.
 super().__init__(iterations)
 def setRate(self):
self.cycle_iterations += 1
if self.cycle_iterations == self.epoch_iterations:
self.cycle_iterations = 0.
 self.epoch_iterations *= self.cycle_mult
decay_phase = np.pi*self.cycle_iterations/self.epoch_iterations
decay =(np.cos(decay_phase)+ 1.) / 2.
 return self.max_lr * decay
def on_train_begin(self, logs={}):
super().on_train_begin(logs={})
 self.cycle_iterations = 0.
 self.max_lr = K.get_value(self.model.optimizer.lr )
result = train_targets_scored.drop(columns=target_cols)\ 
.merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = result[target_cols].values
score = 0
for i in range(y_true.shape[1]):
_score = log_loss(y_true[:,i], y_pred[:,i])
 score += _score / y_true.shape[1]
logger.info(f"Final result: {score}" )<save_to_csv>
def build_model(optimizer=Adam()):
model = Sequential()
model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256))
model.add(BatchNormalization())
 model.add(Dense(128))
model.add(BatchNormalization())
 model.add(Dense(10, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
model = build_model(Adam(learning_rate=1e-3))
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import datetime, tqdm
from kaggle.competitions import nflrush
from sklearn.model_selection import KFold, RepeatedKFold<find_best_params>
cyclic_lr = LR_Cycle(59, cycle_mult=2 )
env = nflrush.make_env()<load_from_csv>
history = model.fit_generator(train_generator,
epochs=31,
callbacks=[cyclic_lr] )
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
 train.head()<drop_column>
test_path = Path.cwd() /'.. '/'input'/'Kannada-MNIST'/'test.csv'
all_data_test = pd.read_csv(test_path )
train.drop(['Season', 'DefensePersonnel', 'Temperature', 'Humidity'], axis = 1, inplace = True )<categorify>
x_test = all_data_test.iloc[:,1:].values
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1 )
off_form = train['OffenseFormation'].unique()
train = pd.concat([train.drop(['OffenseFormation'], axis=1), pd.get_dummies(train['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train.columns<feature_engineering>
predictions = model.predict_classes(x_test/255.)
def fe(train):
train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']
def strtoseconds(txt):
txt = txt.split(':')
 return int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
train['JerseyNumber_grouped'] = train['JerseyNumber'] // 10
train['GameClock'] = train['GameClock'].apply(strtoseconds)
 train['Height_feet'] = train['PlayerHeight'].apply(lambda x: int(x.split('-')[0]))
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['BMI'] =(train['PlayerWeight'] * 703)/(train['PlayerHeight'] ** 2)
 train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1)
 def give_me_WindSpeed(txt):
txt = str(txt ).lower().replace('mph', '' ).strip()
if '-' in txt:
txt =(int(txt.split('-')[0])+ int(txt.split('-')[1])) / 2
try:
return float(txt)
 except:
return -1
train['WindSpeed'] = train['WindSpeed'].apply(give_me_WindSpeed)
 train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right')
 train['Team'] = train['Team'].apply(lambda x: x.strip() =='home')
 train['Field_eq_Possession'] = train['FieldPosition'] == train['PossessionTeam']
def give_me_GameWeather(txt):
txt = str(txt ).lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear')
 txt = txt.replace('skies', '' ).replace("mostly", "" ).strip()
if "indoor" in txt:
txt = "indoor"
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*5
if 'sunny' in txt or 'sun' in txt:
return ans*3
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -3*ans
if 'snow' in txt:
return -5*ans
return 0
train['GameWeather'] = train['GameWeather'].apply(give_me_GameWeather)
 train['IsRusher'] = train['NflId'] == train['NflIdRusher']
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
train.drop(['GameId', 'PlayId', 'index', 'IsRusher', 'Team', 'NflId', 'NflIdRusher'], axis=1, inplace=True)
 cat_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(col)
 train = train.drop(cat_features, axis=1)
 train.fillna(-999, inplace=True)
 return train<feature_engineering>
output = pd.DataFrame({'id': all_data_test.id,
'label': predictions})
 output.to_csv("submission.csv",index=False )