Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<prepare_x_and_y>
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.externals import joblib
import xgboost as xgb
import lightgbm as lgb
X_train = all_data[all_data['target'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['target'].isnull() ].drop(['target'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
X_train.drop(['id'], axis=1, inplace=True)
 X_test_ID = X_test.pop('id')
 Y_train = X_train.pop('target')
 print(X_train.shape, X_test.shape )<prepare_x_and_y>
train_file = '.. /input/train.csv'
test_file = '.. /input/test.csv'
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 splits = 11
for i in tqdm(range(512)) :
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[train2.index]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = sel.fit_transform(train2)
 test2 = sel.transform(test2)
 skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train2[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] = clf.predict_proba(train2[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits<compute_test_metric>
train = pd.read_csv(train_file)
 test = pd.read_csv(test_file )
error = X_train[(( Y_train == 1)&(oof_preds < 0.5)) |(( Y_train == 0)&(oof_preds >= 0.5)) ]
print('error rate: %.3f%%' %(len(error)/ X_train.shape[0] * 100))<concatenate>
test_ID = test['ID']
y_train = train['target']
y_train = np.log1p(y_train )
X_test_p1 = X_test[(sub_preds <= 0.01)].copy()
X_test_p2 = X_test[(sub_preds >= 0.99)].copy()
X_test_p1['target'] = 0
X_test_p2['target'] = 1
print(X_test_p1.shape, X_test_p2.shape)
 Y_train = pd.concat([Y_train, X_test_p1.pop('target'), X_test_p2.pop('target')], axis=0)
 X_train = pd.concat([X_train, X_test_p1, X_test_p2], axis=0)
 Y_train.reset_index(drop=True, inplace=True)
 X_train.reset_index(drop=True, inplace=True )<categorify>
train.drop("ID", axis = 1, inplace = True)
 train.drop("target", axis = 1, inplace = True)
 test.drop("ID", axis = 1, inplace = True )
_=<concatenate>
NUM_OF_DECIMALS = 32
train = train.round(NUM_OF_DECIMALS)
 test = test.round(NUM_OF_DECIMALS )
splits = 11
Y_train_org = Y_train.copy()
X_train_org = X_train.copy()
for itr in range(4):
X_test_p1 = X_test[(sub_preds <= 0.04)].copy()
X_test_p2 = X_test[(sub_preds >= 0.96)].copy()
X_test_p1['target'] = 0
X_test_p2['target'] = 1
print(X_test_p1.shape, X_test_p2.shape)
 Y_train = pd.concat([Y_train_org, X_test_p1.pop('target'), X_test_p2.pop('target')], axis=0)
 X_train = pd.concat([X_train_org, X_test_p1, X_test_p2], axis=0)
 Y_train.reset_index(drop=True, inplace=True)
 X_train.reset_index(drop=True, inplace=True)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for i in tqdm(range(512)) :
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[train2.index]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = pd.DataFrame(sel.fit_transform(train2))
test2 = pd.DataFrame(sel.transform(test2))
sel = StandardScaler()
train2 = pd.DataFrame(sel.fit_transform(train2))
test2 = pd.DataFrame(sel.transform(test2))
skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=42, nu=0.59, coef0=0.053)
 clf.fit(train2.iloc[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] += clf.predict_proba(train2.iloc[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits<save_to_csv>
train_zeros = pd.DataFrame({'Percent_zero':(( train.values)==0 ).mean(axis=0),
'Column' : train.columns})
 high_vol_columns = train_zeros['Column'][train_zeros['Percent_zero'] < 0.70].values
low_vol_columns = train_zeros['Column'][train_zeros['Percent_zero'] >= 0.70].values
submission = pd.DataFrame({
'id': X_test_ID,
'target': sub_preds
})
 submission.to_csv("submission.csv", index=False )<import_modules>
train = train.replace({0:np.nan})
 test = test.replace({0:np.nan} )
from PIL import Image
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, Subset, TensorDataset
from sklearn.model_selection import train_test_split
from tqdm import notebook
import torchvision
from torchvision import transforms
from torchvision import models
from collections import Counter
from pathlib import Path
from sklearn.metrics import confusion_matrix, recall_score
import seaborn as sn
import pyarrow.parquet as pq
from skimage.filters import threshold_otsu
from skimage.transform import AffineTransform, SimilarityTransform, warp, resize<set_options>
cluster_sets = {"low":low_vol_columns, "high":high_vol_columns}
for cluster_key in cluster_sets:
for df in [train,test]:
df["count_not0_"+cluster_key] = df[cluster_sets[cluster_key]].count(axis=1)
 df["sum_"+cluster_key] = df[cluster_sets[cluster_key]].sum(axis=1)
 df["var_"+cluster_key] = df[cluster_sets[cluster_key]].var(axis=1)
 df["median_"+cluster_key] = df[cluster_sets[cluster_key]].median(axis=1)
 df["mean_"+cluster_key] = df[cluster_sets[cluster_key]].mean(axis=1)
 df["std_"+cluster_key] = df[cluster_sets[cluster_key]].std(axis=1)
 df["max_"+cluster_key] = df[cluster_sets[cluster_key]].max(axis=1)
 df["min_"+cluster_key] = df[cluster_sets[cluster_key]].min(axis=1)
 df["skew_"+cluster_key] = df[cluster_sets[cluster_key]].skew(axis=1)
 df["kurtosis_"+cluster_key] = df[cluster_sets[cluster_key]].kurtosis(axis=1)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<load_from_csv>
train_more_simplified = train.drop(high_vol_columns,axis=1 ).drop(low_vol_columns,axis=1)
 test_more_simplified = test.drop(high_vol_columns,axis=1 ).drop(low_vol_columns,axis=1 )
df_test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv' )<categorify>
def run_xgb(train_X, train_y, val_X, val_y, test_X):
params = {'objective': 'reg:linear',
'eval_metric': 'rmse',
'eta': 0.001,
'max_depth': 6,
'subsample': 0.6,
'colsample_bytree': 0.6,
'alpha':0.001,
'random_state': 42,
'silent': True}
print("Load matrices")
 tr_data = xgb.DMatrix(train_X, train_y)
 va_data = xgb.DMatrix(val_X, val_y)
 print("Set watchlist")
 watchlist = [(tr_data, 'train'),(va_data, 'valid')]
print("Train model")
 model_xgb = xgb.train(params, tr_data, 20000, watchlist, maximize=False, early_stopping_rounds = 100, verbose_eval=100)
 dtest = xgb.DMatrix(test_X)
 xgb_pred_y = np.expm1(model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit))
return xgb_pred_y, model_xgb
def thresh(t):
t = t.clone().detach()
t.mul_(-1)
 nn.functional.threshold_(t, -200, -255)
 t.mul_(-1)
 return t
def make_tensordataset_from_dfs(parquet_locs, label_loc=None):
ids = []
X = []
for parquet_loc in parquet_locs:
df = pd.read_parquet(parquet_loc)
 ids.extend(df.image_id.tolist())
 x = df.iloc[:, 1:].to_numpy(dtype=np.uint8)
 del df
X.append(x)
 X = np.vstack(X)
 X = X.reshape(-1, 1, 137, 236)
 X = torch.from_numpy(X)
 ids = dict(( s,i)for(i,s)in enumerate(ids))
if label_loc is None:
return TensorDataset(X)
 else:
graphemes = torch.zeros(X.shape[0], dtype=torch.long)
 vowel_diacs = torch.zeros(X.shape[0], dtype=torch.long)
 consonant_diacs = torch.zeros(X.shape[0], dtype=torch.long)
 lbl_df = pd.read_csv(label_loc)
 for row in lbl_df.itertuples() :
if row.image_id not in ids:
continue
idx = ids[row.image_id]
graphemes[idx] = row.grapheme_root
vowel_diacs[idx] = row.vowel_diacritic
consonant_diacs[idx] = row.consonant_diacritic
return TensorDataset(X, graphemes, vowel_diacs, consonant_diacs )<categorify>
dev_X, val_X, dev_y, val_y = train_test_split(train_more_simplified, y_train, test_size = 0.2, random_state = 40 )
class DatasetWithImageTransforms(Dataset):
def __init__(self, ds, transforms):
super(DatasetWithImageTransforms, self ).__init__()
self.ds = ds
self.tr = transforms
self.nt = len(self.ds[0])
 self.ln = len(self.ds)
 def __getitem__(self, index):
img, *rest = self.ds[index]
img = self.tr(img)
 return(img,)+ tuple(rest)
 def __len__(self):
return self.ln<categorify>
pred_test_xgb, model_xgb = run_xgb(dev_X, dev_y, val_X, val_y, test_more_simplified)
 print("Finished!" )
def thresh(img):
thresh_val = int(threshold_otsu(img))
img =(img > thresh_val)
 return img
def bounding_box(img):
img = thresh(img ).astype(np.uint8)
 col_min_val = np.min(img, axis=0)
 row_min_val = np.min(img, axis=1)
 col = np.argwhere(1 - col_min_val ).flatten()
row = np.argwhere(1 - row_min_val ).flatten()
return row.min() , row.max() , col.min() , col.max()
def scale_to_bb(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
t, l = max(0, t - 10), max(0, l - 10)
 b, r = min(height, b + 10), min(width, r + 10)
 img = resize(img[t:b, l:r], output_shape=(256, 256), preserve_range=True, order=3, cval=1.0)
 return img
def random_scale(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
max_width_scale =(box_width + min(l, width - r)) / box_width
max_height_scale =(box_height + min(t, height - b)) / box_height
max_scale = min(max_width_scale, max_height_scale)
 min_scale = min(1.0, 0.25 * max(( height / box_height),(width / box_width)))
 scale = random.uniform(min_scale, max_scale)
 tfm = SimilarityTransform(
 scale=(scale, scale), 
)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def random_translate(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
translate_height = random.uniform(-t, height - b)
 translate_width = random.uniform(-l, width - r)
 tfm = SimilarityTransform(
 translation=(translate_width, translate_height), 
)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def random_rotate_and_shear(img):
max_theta = math.pi / 16
theta = random.uniform(-max_theta, max_theta)
 max_shear_theta = math.pi / 8
shear_theta = random.uniform(-max_shear_theta, max_shear_theta)
 tfm = AffineTransform(rotation=theta, shear=shear_theta)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def invert_color(t):
t.mul_(-1)
 t.add_(255)
 return t
def affine_transforms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = random_translate(random_scale(img))
img = img.reshape(1, 137, 236)
 return torch.from_numpy(img)
 def tfms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = scale_to_bb(img)
 img = img.reshape(1, 256, 256)
 return torch.from_numpy(img)
 def va_tfms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = img.reshape(1, 137, 236)
 return torch.from_numpy(img)
 affine_transforms = transforms.Lambda(affine_transforms)
 mult = transforms.Lambda(lambda img: img * 255)
 to_float = transforms.Lambda(lambda img: img.float())
 invert_color = transforms.Lambda(invert_color)
 tfms = transforms.Compose([
tfms,
to_float,
] )<define_variables>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub["target"] = pred_test_xgb
<init_hyperparams><EOS> 
sub.to_csv('sub_XGB_Aggregate_v2.csv', index=False )