Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<load_pretrained>
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import LeaveOneOut
from sklearn.feature_selection import VarianceThreshold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm_notebook as tqdm
from sklearn.mixture import GaussianMixture as GM
train_loader = DataLoader(train, batch_size=batch_size_train,num_workers=2, shuffle=True)
 valid_loader = DataLoader(valid, batch_size=batch_size_test, num_workers=2, shuffle=True)
 test_loader = DataLoader(test, batch_size=batch_size_test, shuffle=False )<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.conv_block = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2)
)
 self.linear_block = nn.Sequential(
 nn.Dropout(p=0.5),
nn.Linear(128*7*7, 128),
nn.BatchNorm1d(128),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(128, 64),
nn.BatchNorm1d(64),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(64, 10)
)
 def forward(self, x):
x = self.conv_block(x)
 x = x.view(x.size(0), -1)
 x = self.linear_block(x)
 return x<choose_model_class>
if test.shape[0] < 150000:
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.to_csv('submission.csv',index=False)
 6 + 'j'
cnn_model = Net()
criterion = nn.CrossEntropyLoss()
if torch.cuda.is_available() :
cnn_model.cuda()
criterion.cuda()
optimizer = optim.Adam(params=cnn_model.parameters() , lr=learning_rate)
 exp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min')
 train_losses = []
train_counter = []
test_losses = []
test_counter = [i*len(train_loader.dataset)for i in range(1, num_epoch + 1)]
best_model_wts = copy.deepcopy(cnn_model.state_dict())
 best_acc = 0.0
since = time.time()
for epoch in range(1, num_epoch + 1):
cnn_model.train()
for i,(images, labels)in enumerate(train_loader):
images = Variable(images ).cuda()
labels = Variable(labels ).cuda()
optimizer.zero_grad()
outputs = cnn_model(images)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
if(i + 1)% log_interval == 0:
print('Train Epoch: {} [{}/{}({:.0f}%)]\tLoss: {:.6f}'.format(
 epoch,(i + 1)* len(images), len(train_loader.dataset),
100.*(i + 1)/ len(train_loader), loss.data))
train_losses.append(loss.item())
 train_counter.append(( i*64)+(( epoch-1)*len(train_loader.dataset)))
 cnn_model.eval()
loss = 0
running_corrects = 0
with torch.no_grad() :
for i,(data, target)in enumerate(valid_loader):
data = Variable(data ).cuda()
target = Variable(target ).cuda()
output = cnn_model(data)
 loss += F.cross_entropy(output, target, reduction='sum' ).item()
_, preds = torch.max(output, 1)
 running_corrects += torch.sum(preds == target.data)
 loss /= len(valid_loader.dataset)
 test_losses.append(loss)
 epoch_acc = 100.* running_corrects.double() / len(valid_loader.dataset)
 print('
Average Val Loss: {:.4f}, Val Accuracy: {}/{}({:.3f}%)
 '.format(
 loss, running_corrects, len(valid_loader.dataset), epoch_acc))
if epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(cnn_model.state_dict())
 exp_lr_scheduler.step(loss)
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))<predict_on_test>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
shapes = []
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 full = np.vstack([train3, test3])
 shapes.append(train3.shape[1])
 n_components = 6
n_init = 10
best_reg_covar = {
33: 1.5,
34: 2.,
35: 2.5,
36: 3.5,
37: 4.,
38: 4.5,
39: 5.,
40: 5.5,
41: 6.,
42: 6.5,
43: 7.,
44: 7.5,
45: 8.,
46: 8.5,
47: 9.
 }
reg_covar = best_reg_covar[train3.shape[1]]
X = np.vstack([train3, test3])
 gm = GM(n_components=n_components, n_init=n_init,
random_state=0, init_params='random', reg_covar=reg_covar)
 gm.fit(X)
 res = pd.DataFrame(data=gm.predict(train3), columns=['cluster'])
 res['target'] = train2['target'].values
res = res.groupby('cluster')['target'].mean()
ones_indices = []
for j in range(len(res)) :
if res[j] > 0.5:
ones_indices.append(j)
 oof[idx1] = gm.predict_proba(train3)[:, ones_indices].sum(axis=1)
 preds[idx2] = gm.predict_proba(test3)[:, ones_indices].sum(axis=1)
 auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))
<save_to_csv><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()