Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<choose_model_class>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense,Conv2D,Flatten,MaxPool2D,Dropout,BatchNormalization
from keras.optimizers import RMSprop,Adam
from keras.callbacks import ReduceLROnPlateau
model = Model(num_features, num_all_targets)
 model<define_search_model>
root = ".. /input/Kannada-MNIST"
train = pd.read_csv(os.path.join(root,'train.csv'))
test = pd.read_csv(os.path.join(root,'test.csv'))
final = pd.read_csv(os.path.join(root,'sample_submission.csv'))
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<train_model>
test = test.drop('id',axis=1 )
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
X_train=train.drop('label',axis=1)
 Y_train=train.label
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
X_train=X_train/255
test=test/255
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions<feature_engineering>
Y_train=to_categorical(Y_train )
str(timedelta(seconds=time_diff))<compute_test_metric>
X_train,X_test,y_train,y_test=train_test_split(X_train,Y_train,random_state=42,test_size=0.2 )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<save_to_csv>
data_tweaker = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 data_tweaker.fit(X_train )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<set_options>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization(momentum=.15))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization(momentum=0.15))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization(momentum=.15))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.4))
model.add(Dense(10, activation = "softmax"))
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<feature_engineering>
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
 model.compile(optimizer=optimizer,loss=['categorical_crossentropy'],metrics=['accuracy'] )
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<load_from_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<merge>
epochs=30
batch_size=64
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )<categorify>
train_info = model.fit_generator(data_tweaker.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_test,y_test),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )<prepare_x_and_y>
test_data = pd.read_csv(os.path.join(root,'test.csv'))
test_data = test_data.drop('id',axis=1)
 test_data = test_data/255
test_data = test_data.values.reshape(-1,28,28,1 )
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x<categorify>
pred = model.predict(test_data)
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )<init_hyperparams>
final['label'] = np.argmax(pred, axis=1)
 final['id'] = np.arange(len(final['label']))
final.to_csv('.. /working/submission.csv',index=False )