Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<set_options>
torch.manual_seed(0)
 np.random.seed(0)
 TRAIN_PATH = '.. /input/Kannada-MNIST/train.csv'
TEST_PATH = '.. /input/Kannada-MNIST/test.csv'
SUBMISSION_PATH = '/kaggle/working/submission.csv'
DIG_PATH = '.. /input/Kannada-MNIST/Dig-MNIST.csv'
%who DataFrame
del test, train
gc.collect()
%who DataFrame<load_from_csv>
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,predictors,cs,gs 
<compute_test_metric>
class KannadaMNISTDataset(Dataset):
def __init__(self, csv_path, transform):
data = pd.read_csv(csv_path ).to_numpy()
self.y = data[:, 0]
self.X = data[:, 1:].reshape(-1, 28, 28 ).astype(np.uint8)
 assert self.y.shape[0] == self.X.shape[0]
self.transform = transform
def __len__(self):
return self.X.shape[0]
def __getitem__(self, idx):
return self.transform(self.X[idx]), self.y[idx]
def asPIL(self, idx):
return transforms.ToPILImage()(self.X[idx]), self.y[idx]
def merge(self, other):
self.X = np.concatenate(( self.X, other.X))
self.y = np.concatenate(( self.y, other.y))
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test 
<choose_model_class>
transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.Resize(( 32, 32)) ,
transforms.RandomAffine(degrees=10, translate=(0.25, 0.25), scale=(0.75, 1.25), shear=0.1),
transforms.ToTensor()
])
 dataset = KannadaMNISTDataset(TRAIN_PATH, transform)
 dataset.merge(KannadaMNISTDataset(DIG_PATH, transform))
train_idx, test_idx = train_test_split(range(len(dataset)) , train_size=0.85)
 train_sampler = SubsetRandomSampler(train_idx)
 test_sampler = SubsetRandomSampler(test_idx)
 BATCH_SIZE = 512
train_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=8, pin_memory=True)
 test_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, sampler=test_sampler, num_workers=8, pin_memory=True )
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<find_best_model_class>
class SimpleNet(nn.Module):
def __init__(self):
super(SimpleNet, self ).__init__()
def make_conv_group(in_, out_):
return nn.Sequential(
 nn.Conv2d(in_, out_, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) ,
nn.BatchNorm2d(out_, momentum=0.05),
nn.ReLU(inplace=True)
)
 self.layers = nn.Sequential(
 make_conv_group(1, 64),
make_conv_group(64, 128),
make_conv_group(128, 128),
make_conv_group(128, 128),
nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2)) ,
nn.Dropout2d(p=0.1),
make_conv_group(128, 128),
make_conv_group(128, 128),
make_conv_group(128, 256),
nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2)) ,
nn.Dropout2d(p=0.1),
make_conv_group(256, 256),
make_conv_group(256, 256),
nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2)) ,
nn.Dropout2d(p=0.1),
make_conv_group(256, 512),
nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2)) ,
nn.Dropout2d(p=0.1),
make_conv_group(512, 2048),
make_conv_group(2048, 256),
nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2)) ,
nn.Dropout2d(p=0.1),
make_conv_group(256, 256)
)
 self.head = nn.Linear(256, 10)
 for layer in self.layers.modules() :
if isinstance(layer, nn.Conv2d):
nn.init.xavier_uniform_(layer.weight.data, gain=nn.init.calculate_gain('relu'))
def forward(self, x):
x = self.layers(x)
 x = x.view(x.size(0), -1)
 x = self.head(x)
 x = nn.Softmax(dim=1 )(x)
 return x
def predict_resnet(train_features,test_features,labels_train, SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
_,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 data_test_2 = test_features[predictors].values
for seed in SEEDS:
for fold in range(NFOLDS):
K.clear_session()
model = keras.models.load_model('/kaggle/input/resent-1841/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 print(f"Pretrained_Resnet_Model: {seed}, FOLD: {fold} ")
 return y_pred 
<predict_on_test>
net = SimpleNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters())
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', threshold=1e-5, verbose=True)
 EPOCH = 0
train_features,labels_train,test_features,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
test_pred_rs = predict_resnet(train_features,test_features,labels_train,SEEDS, NFOLDS,cs,gs )<save_to_csv>
for epoch in tqdm(range(EPOCH, EPOCH + 100), leave=False):
net.train()
for batch_x, batch_y in tqdm(train_loader, leave=False):
optimizer.zero_grad()
outputs = net(batch_x.cuda())
 loss = criterion(outputs, batch_y.cuda())
 loss.backward()
optimizer.step()
test_loss = 0.0
net.eval()
for batch_x, batch_y in tqdm(test_loader, leave=False):
outputs = net(batch_x.cuda())
 loss = criterion(outputs, batch_y.cuda())
 test_loss += loss.item()
test_loss /= len(test_idx)
 scheduler.step(test_loss )
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.zeros([test_pred_rs.shape[0],test_pred_rs.shape[1]])
 sub_rs.iloc[:,1:] = test_pred_rs
sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False)
<create_dataframe>
test_dataset = KannadaMNISTDataset(TEST_PATH, transform)
 loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE, num_workers=8, pin_memory=True)
 test_answers = None
net.eval()
for batch_x, _ in tqdm(loader):
outputs = net(batch_x.cuda() ).cpu().detach().numpy()
if test_answers is not None:
test_answers = np.concatenate(( test_answers, outputs))
else:
test_answers = outputs
good_ind = test_answers.max(axis=1)>= 0.95
good_labels = test_answers[good_ind].argmax(axis=1 )
%who DataFrame
del train_features,labels_train,test_features,predictors,cs,gs
gc.collect()
%who DataFrame<normalization>
pseudo_dataset = KannadaMNISTDataset(TEST_PATH, transform)
 pseudo_dataset.X = pseudo_dataset.X[good_ind]
pseudo_dataset.y = good_labels
pseudo_loader = torch.utils.data.DataLoader(pseudo_dataset, batch_size=BATCH_SIZE, num_workers=8, pin_memory=True)
 for lr in [1e-4, 1e-5, 1e-6]:
for g in optimizer.param_groups:
g['lr'] = lr
for epoch in tqdm(range(EPOCH, EPOCH + 100), leave=False):
net.train()
for batch_x, batch_y in tqdm(pseudo_loader, leave=False):
optimizer.zero_grad()
outputs = net(batch_x.cuda())
 loss = criterion(outputs, batch_y.cuda())
 loss.backward()
optimizer.step()
EPOCH += 1
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def remove_ctl(no_ctl,train,test,targets):
if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True)
 return train,test,targets
def remove_small_variance(variance_threshould,train,test):
cols_numeric = [feat for feat in list(train.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp_train = train[cols_numeric].loc[:, mask]
tmp_test = test[cols_numeric].loc[:, mask]
train = pd.concat([train[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_train], axis=1)
 test = pd.concat([test[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_test], axis=1)
 return train,test
def pca_process(ncompo_genes,ncompo_cells,train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit(train[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit(train[CELLS])
 train_pca_gene = pca_genes.transform(train[GENES])
 test_pca_gene = pca_genes.transform(test[GENES])
 train_pca_cell = pca_cells.transform(train[CELLS])
 test_pca_cell = pca_cells.transform(test[CELLS])
 train_pca_gene = pd.DataFrame(train_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 test_pca_gene = pd.DataFrame(test_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 train_pca_cell = pd.DataFrame(train_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 test_pca_cell = pd.DataFrame(test_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 train_df = pd.concat([train, train_pca_gene, train_pca_cell], axis=1)
 test_df = pd.concat([test, test_pca_gene, test_pca_cell], axis=1)
 return train_df,test_df
def scale_data(scale, train_df,test_df):
if scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 cols_numeric = [feat for feat in list(train_df.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose'] and "pca_" not in feat]
scaler = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train_df[cols_numeric] = scaler.fit_transform(train_df[cols_numeric])
 test_df[cols_numeric] = scaler.transform(test_df[cols_numeric])
 else:
pass
return train_df,test_df
def add_features(train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
train['g_'+stats] = getattr(train[GENES], stats )(axis=1)
 train['c_'+stats] = getattr(train[CELLS], stats )(axis=1)
 train['gc_'+stats] = getattr(train[GENES+CELLS], stats )(axis=1)
 test['g_'+stats] = getattr(test[GENES], stats )(axis=1)
 test['c_'+stats] = getattr(test[CELLS], stats )(axis=1)
 test['gc_'+stats] = getattr(test[GENES+CELLS], stats )(axis=1)
 return train,test
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def data_processing(train,test, targets):
train,test,targets = remove_ctl(no_ctl,train,test,targets)
 print("remove control: ", train.shape,test.shape)
 train,test = remove_small_variance(variance_threshould,train,test)
 print("remove small_var: ", train.shape,test.shape)
 train_df,test_df = pca_process(ncompo_genes,ncompo_cells,train,test)
 print("do PCA: ", train_df.shape,test_df.shape)
 train_df,test_df = scale_data(scale, train_df,test_df)
 print("rankGauss: ", train_df.shape,test_df.shape)
 train_df,test_df = add_features(train_df,test_df)
 print("add stats feature:", train_df.shape,test_df.shape)
 return train_df,test_df,targets<groupby>
test_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.Resize(( 32, 32)) ,
transforms.ToTensor()
])
 test_dataset = KannadaMNISTDataset(TEST_PATH, test_transform)
 loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE)
 results = []
net.eval()
for batch_x, _ in tqdm(loader):
outputs = net(batch_x.cuda())
 results.extend(map(lambda x: x.item() , outputs.argmax(dim=1)))
 submission = pd.DataFrame({
'id': range(len(test_dataset)) ,
'label': results
})
 submission.to_csv(SUBMISSION_PATH, index=False )