Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<choose_model_class>
py.init_notebook_mode(connected=True)
 plt.style.use('ggplot')
 sns.set(font_scale=1)
 pd.set_option('display.max_columns', 500 )
def segnet_mod() :
kernel = 3
pad = 1
pool_size = 2
model = Sequential()
model.add(Layer(input_shape=(32, 32 ,3)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(64, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(128, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(256, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(512, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
 model.add(UpSampling2D(size=(pool_size,pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(512, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(UpSampling2D(size=(pool_size,pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(256, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(UpSampling2D(size=(pool_size,pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(128, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(UpSampling2D(size=(pool_size,pool_size)))
 model.add(ZeroPadding2D(padding=(pad,pad)))
 model.add(Conv2D(64, kernel, kernel, border_mode='valid'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=Adam(lr=0.001),
loss='binary_crossentropy',
metrics=['accuracy'])
 return model<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def image_batch_generator(img_paths, set_type, batchsize=32):
while True:
ig = image_gen(img_paths)
 batch_img, batch_label = [], []
for img, label in ig:
if set_type == 'train':
vertical = vertical_augmentation(img)
 vertical =(vertical - vertical.mean())/vertical.std()
batch_img.append(vertical)
 batch_label.append(label)
 img =(img - img.mean())/img.std()
batch_img.append(img)
 batch_label.append(label)
 if len(batch_img)== batchsize:
yield np.stack(batch_img, axis=0), np.stack(batch_label, axis=0)
 batch_img, batch_label = [], []
if len(batch_img)!= 0:
yield np.stack(batch_img, axis=0), np.stack(batch_label, axis=0)
 batch_img, batch_label = [], []<import_modules>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
device_lib.list_local_devices()<define_variables>
features = [c for c in train.columns if c not in ['ID_code', 'target']]
features_t = [c for c in test.columns if c not in ['ID_code']]
y = train.target
BATCHSIZE = 32
train_img_paths, val_img_paths = train_test_split(train_img_paths, test_size=0.10)
 traingen = image_batch_generator(train_img_paths,'val', batchsize=BATCHSIZE)
 valgen = image_batch_generator(val_img_paths, 'val', batchsize=BATCHSIZE)
 def calc_steps(data_len, batchsize):
return(data_len + batchsize - 1)// batchsize
train_steps = calc_steps(len(train_img_paths), BATCHSIZE)
 val_steps = calc_steps(len(val_img_paths), BATCHSIZE)
 history = model.fit_generator(
 traingen,
steps_per_epoch=train_steps,
epochs = 20,
validation_data=valgen,
validation_steps=val_steps,
verbose=1,
max_queue_size=1
)<predict_on_test>
%%time
def clusters(train,y):
distance = []
for cluster in range(2,11,1):
plt.figure(figsize=(10,7))
print('Started checking {}'.format(cluster))
kmeans = KMeans(cluster, random_state=2702)
 labels = kmeans.fit_predict(train)
 plt.title('Clusters {}'.format(cluster))
sns.countplot(labels, hue=y)
 plt.show()
distance.append(kmeans.inertia_)
 gc.collect()
return distance
train_dist = clusters(train[features],train.target )
def generate_predictions_generator(val_paths):
for img_path in val_paths:
img = imread(img_path)
 img_id = get_img_basename(img_path)
 img =(img - img.mean())/img.std()
y = model.predict(img.reshape(1,32,32,3))
yield img_id, y
def create_submission(csv_name, predictions_gen):
sub = pd.DataFrame()
ids = []
encodings = []
num_images = len(test_img_paths)
 for i in range(num_images):
if(i+1)%(num_images//10)== 0:
print(i, num_images)
 img_id, pred = next(predictions_gen)
 ids.append(img_id)
 encodings.append(pred[0][0])
 sub['id'] = ids
sub['has_cactus'] = encodings
sub.to_csv(csv_name, index=False)
 tic = time.time()
create_submission("cactus.csv", generate_predictions_generator(test_img_paths))
toc = time.time()
print(toc - tic)
<define_variables>
sc = StandardScaler(copy=False)
 train_sc = pd.DataFrame(sc.fit_transform(train[features]), columns=features)
 test_sc = pd.DataFrame(sc.fit_transform(test[features_t]), columns=features_t)
 gc.collect()
data_dir = '.. /input'
train_dir = data_dir + '/train/train/'
test_dir = data_dir + '/test/test/'<load_from_csv>
mn = MinMaxScaler(copy=False)
 train_mn = pd.DataFrame(mn.fit_transform(train[features]), columns=features)
 test_mn = pd.DataFrame(mn.fit_transform(test[features_t]), columns=features_t)
 gc.collect()
labels = pd.read_csv(".. /input/train.csv")
 labels.head()<count_values>
qt = QuantileTransformer(n_quantiles = 200)
 train_qt = pd.DataFrame(mn.fit_transform(train[features]), columns=features)
 test_qt = pd.DataFrame(mn.fit_transform(test[features_t]), columns=features_t)
 gc.collect()
balance = labels['has_cactus'].value_counts()
print(balance )<split>
rs = RobustScaler(copy=False)
 train_rs = pd.DataFrame(rs.fit_transform(train[features]), columns=features)
 test_rs = pd.DataFrame(rs.fit_transform(test[features_t]), columns=features_t)
 gc.collect()
train, valid = train_test_split(labels, stratify=labels.has_cactus, test_size=0.2 )<train_model>
X_train, X_test, y_train, y_test = train_test_split(
 train[features], y, test_size=0.3,stratify = y, random_state=2701)
 model = lgb.LGBMClassifier(
 n_estimators = 5000,
learning_rate= 0.1,
metric='auc', 
)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )
print('Number of training samples:{0}'.format(len(train)))
 print('Number of validation samples:{0}'.format(len(valid)))
 print('Number of testing samples:{0}'.format(len(os.listdir(test_dir))))<init_hyperparams>
pred = model.predict(X_test)
 print(f'AUC: {roc_auc_score(pred, y_test)}' )
num_epochs = 25
num_classes = 2
batch_size = 128
learning_rate = 0.002
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<categorify>
X_train, X_test, y_train, y_test = train_test_split(
 train_sc[features], y, test_size=0.3,stratify = y, random_state=2701)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )
class CactiImageData(Dataset):
def __init__(self, split_data, data_root = './', transform=None):
super().__init__()
self.df = split_data.values
self.data_root = data_root
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name,label = self.df[index]
img_path = os.path.join(self.data_root, img_name)
 image = mpimg.imread(img_path)
 if self.transform is not None:
image = self.transform(image)
 return image, label<categorify>
pred = model.predict(X_test)
 print(f'AUC: {roc_auc_score(pred, y_test)}' )
mean = [0.5, 0.5, 0.5]
std = [0.5, 0.5, 0.5]
train_transf = transforms.Compose([transforms.ToPILImage() ,
transforms.ToTensor() ])
 valid_transf = transforms.Compose([transforms.ToPILImage() ,
transforms.ToTensor() ] )<load_pretrained>
X_train, X_test, y_train, y_test = train_test_split(
 train_qt[features], y, test_size=0.3,stratify = y, random_state=2701)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )
train_data = CactiImageData(split_data = train, data_root = train_dir, transform = train_transf)
 valid_data = CactiImageData(split_data = valid, data_root = train_dir, transform = valid_transf)
 train_loader = DataLoader(dataset = train_data, batch_size = batch_size, shuffle = True, num_workers = 0)
 valid_loader = DataLoader(dataset = valid_data, batch_size = batch_size//2, shuffle = False, num_workers = 0 )<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(
 train_rs[features], y, test_size=0.3,stratify = y, random_state=2701)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )
class CactiCNN(nn.Module):
def __init__(self):
super(CactiCNN, self ).__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels=3,out_channels=32,kernel_size=3,stride=1,padding=0),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2))
self.conv2 = nn.Sequential(
 nn.Conv2d(in_channels=32,out_channels=64,kernel_size=2,stride=1,padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2))
self.conv3 = nn.Sequential(
 nn.Conv2d(in_channels=64,out_channels=128,kernel_size=3,stride=1,padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2))
self.conv4 = nn.Sequential(
 nn.Conv2d(in_channels=128,out_channels=256,kernel_size=3,stride=1,padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2))
self.fc = nn.Sequential(
 nn.Linear(256 * 2 * 2, 1024),
nn.ReLU(inplace=True),
nn.Dropout(0.25),
nn.Linear(1024, 2))
def forward(self,x):
x = self.conv1(x)
 x = self.conv2(x)
 x = self.conv3(x)
 x = self.conv4(x)
 x = x.view(x.shape[0],-1)
 x = self.fc(x)
 return x<choose_model_class>
pred = model.predict(X_test)
 print(f'AUC: {roc_auc_score(pred, y_test)}' )
model = CactiCNN().to(device)
 criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate )<choose_model_class>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}
total_step = len(train_loader)
 for epoch in range(num_epochs):
for i,(images, labels)in enumerate(train_loader):
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 loss = criterion(outputs, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(i+1)% 100 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' 
.format(epoch+1, num_epochs, i+1, total_step, loss.item()))<predict_on_test>
num_folds = 11
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=2702)
 oof = np.zeros(len(train))
predictions = np.zeros(len(y))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, y.values)) :
print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=y.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=2000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(y, oof)) )
model.eval()
with torch.no_grad() :
correct = 0
total = 0
for images, labels in valid_loader:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
print('Test Accuracy: {} %'.format(100 * correct / total))<load_from_csv>
sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions
sub.to_csv('submission.csv', index=False )