Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<load_from_csv>
warnings.filterwarnings('ignore' )
train = np.loadtxt('.. /input/digit-recognizer/train.csv', delimiter=',', skiprows=1, dtype=np.uint8)
 train_X = train[:, 1:].reshape(-1, 28, 28, 1)
 train_y = train[:, 0].astype(np.int)
 is_dev = np.arange(train_X.shape[0])% 10 == 0
dev_X = train_X[is_dev]
dev_y = train_y[is_dev]
train_X = train_X[~is_dev]
train_y = train_y[~is_dev]
X_mean = train_X.astype(np.float32 ).mean() / 255
X_std = train_X.astype(np.float32 ).std() / 255
test_X = np.loadtxt('.. /input/digit-recognizer/test.csv', delimiter=',', skiprows=1, dtype=np.uint8)
 test_X = test_X.reshape(-1, 28, 28, 1)
 print(X_mean, X_std )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )
class TransformDataset(Dataset):
def __init__(self, transform, X, y=None):
self.transform = transform
self.X = X
self.y = y
def __len__(self):
return self.X.shape[0]
def __getitem__(self, index):
X_item = self.transform(self.X[index])
 if self.y is None:
return X_item,
else:
return X_item, self.y[index]
train_transform = Compose([
ToPILImage() ,
Pad(2),
RandomAffine(degrees=30, translate=(0.2, 0.2), shear=0.2),
ToTensor() ,
Normalize(( X_mean,),(X_std,)) ,
RandomErasing()
])
 test_transform = Compose([
ToPILImage() ,
Pad(2),
ToTensor() ,
Normalize(( X_mean,),(X_std,))
])
 train_loader = DataLoader(TransformDataset(train_transform, train_X, train_y), batch_size=100, shuffle=True)
 dev_loader = DataLoader(TransformDataset(test_transform, dev_X, dev_y), batch_size=400, shuffle=False)
 test_loader = DataLoader(TransformDataset(test_transform, test_X), batch_size=400, shuffle=False )<choose_model_class>
oof = np.zeros(len(train))
preds = np.zeros(len(test))
params = [{'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]}]
reg_params = np.zeros(512)
 for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.25)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}' )
model = resnet18(num_classes=10)
 model.conv1 = nn.Conv2d(1, 64, kernel_size=5, stride=1, padding=2, bias=False)
 nn.init.kaiming_normal_(model.conv1.weight, mode='fan_out', nonlinearity='relu')
 model.maxpool = nn.Identity()
criterion = nn.CrossEntropyLoss()<choose_model_class>
train_f = train.copy()
train_f.loc[oof > 0.995, 'target'] = 1
train_f.loc[oof < 0.005, 'target'] = 0
oof_ls = np.zeros(len(train))
pred_te_ls = np.zeros(len(test))
for k in tqdm_notebook(range(512)) :
train2 = train_f[train_f['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2['target']=-1
train2p = pd.concat([train2,test2],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train4p = sel.transform(train2p[cols])
 train4 = sel.transform(train2[cols])
 test4 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train4p, train2p['target']):
test_index3 = test_index[ test_index<len(train4)]
clf = LabelSpreading(gamma=0.01,kernel='rbf', max_iter=10,n_jobs=-1)
 clf.fit(train4p[train_index,:],train2p.loc[train_index]['target'])
 oof_ls[idx1[test_index3]] = clf.predict_proba(train4[test_index3,:])[:,1]
pred_te_ls[test2.index] += clf.predict_proba(test4)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_ls)
 print('CV for LabelSpreading =',round(auc,5))
max_dev_acc = 0
optimizer = torch.optim.SGD(model.parameters() , lr=0.1, momentum=0.9)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', verbose=True, patience=1, factor=0.1**0.5)
 for epoch in tqdm(range(20), desc='Epochs'):
train_loss = train_epoch(model, criterion, optimizer, train_loader)
 acc = classification(model, dev_loader)
 scheduler.step(acc)
 print('trainLoss=%.4f' % train_loss, 'devAcc=%.4f' % acc)
 if acc > max_dev_acc:
max_dev_acc = acc
torch.save(model.state_dict() , 'model.pt' )<save_to_csv>
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps
<install_modules><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()