Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<define_variables>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import gmean
from sklearn.model_selection import cross_val_predict, cross_val_score, StratifiedKFold, StratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline, make_union
from sklearn.preprocessing import OneHotEncoder, QuantileTransformer, MinMaxScaler, StandardScaler
fMRI_train_data_path = '.. /input/trends-assessment-prediction/fMRI_train/'
filenames = random.choices(os.listdir(fMRI_train_data_path), k=4 )<load_from_csv>
ROOT = '/kaggle/input/cat-in-the-dat/'
class FieldNames:
idx = 'id'
bin_0 = 'bin_0'
bin_1 = 'bin_1'
bin_2 = 'bin_2'
bin_3 = 'bin_3'
bin_4 = 'bin_4'
nom_0 = 'nom_0'
nom_1 = 'nom_1'
nom_2 = 'nom_2'
nom_3 = 'nom_3'
nom_4 = 'nom_4'
nom_5 = 'nom_5'
nom_6 = 'nom_6'
nom_7 = 'nom_7'
nom_8 = 'nom_8'
nom_9 = 'nom_9'
ord_0 = 'ord_0'
ord_1 = 'ord_1'
ord_2 = 'ord_2'
ord_3 = 'ord_3'
ord_4 = 'ord_4'
ord_5 = 'ord_5'
ord_5a = 'ord_5a'
ord_5b = 'ord_5b'
day = 'day'
month = 'month'
target = 'target'
class FileNames:
train = 'train.csv'
test = 'test.csv'
train_v1 = 'train_v1.ftr'
test_v1 = 'test_v1.ftr'
sub = 'submission.csv'
class ColumnGroup:
binary = [FieldNames.bin_0, FieldNames.bin_1, FieldNames.bin_2,
FieldNames.bin_3, FieldNames.bin_4]
ordinal = [FieldNames.ord_0, FieldNames.ord_1, FieldNames.ord_2,
FieldNames.ord_3, FieldNames.ord_4, FieldNames.ord_5,
]
nominal = [FieldNames.nom_0, FieldNames.nom_1, FieldNames.nom_2,
FieldNames.nom_3, FieldNames.nom_4, FieldNames.nom_5,
FieldNames.nom_6, FieldNames.nom_7, FieldNames.nom_8,
FieldNames.nom_9]
fnc_df = pd.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = pd.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 labels_df = pd.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df["is_train"] = True
df = df.merge(labels_df, on="Id", how="left")
 target_cols = ['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']
test_df = df[df["is_train"] != True].copy()
train_df = df[df["is_train"] == True].copy()<prepare_x_and_y>

 def _convert_to_2d_array(X):
X = np.array(X)
 if X.ndim == 1:
return X.reshape(-1, 1)
 return X
class BaseTransformer(BaseEstimator, TransformerMixin):

 def fit(self, X, y=None):

 return self
@abstractmethod
def _transform(self, X):
pass
def transform(self, X, y=None):

 return self._transform(X)
 class ArrayTransformer(BaseTransformer):

 def transform(self, X):

 Xt = self._transform(X)
 return _convert_to_2d_array(Xt)
 class SelectCols(BaseTransformer):

 def __init__(self, cols):

 self.cols = cols
def _transform(self, X, y=None):
return X[self.cols]
class LabelEncoderWithThresh(ArrayTransformer):

 def __init__(self, threshold=0):
self.threshold = threshold
self.label2int = []
def fit(self, X, y=None):
self.label2int = []
X = np.asarray(X)
 m = X.shape[1]
if not isinstance(self.threshold, list):
thresholds = [self.threshold]*m
else:
thresholds = self.threshold
for j in range(m):
cnts = Counter(X[:, j])
 self.label2int.append({lbl: i+1 for i,(lbl, cnt)in enumerate(cnts.items())if cnt >
thresholds[j]})
 return self
def _transform(self, X):
X = np.asarray(X)
 Xt = np.zeros_like(X)
 for i in range(len(self.label2int)) :
Xt[:, i] = [self.label2int[i].get(val, 0)for val in X[:, i]]
return Xt
y_train_df = train_df[target_cols]
train_df = train_df.drop(target_cols + ['is_train'], axis=1)
 test_df = test_df.drop(target_cols + ['is_train'], axis=1)
 FNC_SCALE = 1/500
test_df[fnc_features] *= FNC_SCALE
train_df[fnc_features] *= FNC_SCALE<compute_test_metric>
def map_ordinals(df):
ord_1_map = {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4.0}
ord_2_map = {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5.3}
ord_3_map = {char: i for i, char in enumerate(sorted(df.ord_3.unique())) }
ord_4_map = {char: i for i, char in enumerate(sorted(df.ord_4.unique())) }
ord_5_map = {char: i for i, char in enumerate(sorted(df.ord_5.unique())) }
bin_4_map = {'N': 0, 'Y': 1}
bin_3_map = {'F': 0, 'T': 1}
df[FieldNames.ord_1] = df[FieldNames.ord_1].map(ord_1_map)
 df[FieldNames.ord_2] = df[FieldNames.ord_2].map(ord_2_map)
 df[FieldNames.ord_3] = df[FieldNames.ord_3].map(ord_3_map)
 df[FieldNames.ord_4] = df[FieldNames.ord_4].map(ord_4_map)
 df[FieldNames.ord_5a] = df[FieldNames.ord_5].str[0]
df[FieldNames.ord_5b] = df[FieldNames.ord_5].str[1]
ord_5a_map = {char: i for i, char in enumerate(sorted(df.ord_5a.unique())) }
ord_5b_map = {char: i for i, char in enumerate(sorted(df.ord_5b.unique())) }
df[FieldNames.ord_5a] = df[FieldNames.ord_5a].map(ord_5a_map)
 df[FieldNames.ord_5b] = df[FieldNames.ord_5b].map(ord_5b_map)
 df[FieldNames.ord_5] = df[FieldNames.ord_5].map(ord_5_map)
 df[FieldNames.bin_3] = df[FieldNames.bin_3].map(bin_3_map)
 df[FieldNames.bin_4] = df[FieldNames.bin_4].map(bin_4_map)
 return df
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))<split>
tr = pd.read_csv(ROOT + FileNames.train)
 te = pd.read_csv(ROOT + FileNames.test)
 tr = map_ordinals(tr)
 te = map_ordinals(te)
 print('Mapping binary and ordinal columns done.')
 y = tr.target.values.astype(int)
 cvlist = list(StratifiedShuffleSplit(n_splits=5, test_size=0.05, random_state=123457869 ).split(tr, y))
param = {'objective':'regression',
'metric':'rmse',
'bossting_type':'gbdt',
'learning_rate':0.01,
'max_depth':-1}
output = pd.DataFrame()
for target in ['age','domain1_var1','domain1_var2','domain2_var1','domain2_var2']:
X_train, X_val, y_train, y_val = train_test_split(train_df.iloc[:,1:], y_train_df[target], test_size=0.2, shuffle=True, random_state=20)
 train_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_val, label=y_val)
 model = lgb.train(param,
train_data,
10000,
early_stopping_rounds=15,
valid_sets=[val_data],
verbose_eval=50)
 temp = pd.DataFrame(test_df['Id'].apply(lambda x:str(x)+ '_'+ target))
temp['Predicted'] = model.predict(test_df.iloc[:,1:])
 output = pd.concat([output,temp] )<load_from_csv>
qtr_map = {1:1, 2:1, 3:1, 4:2, 5:2, 6:2, 7:3, 8:3, 9:3, 10:4, 11:4, 12:4}
tr['qtr'] = tr['month'].map(qtr_map)
 te['qtr'] = te['month'].map(qtr_map )
sample_submission = pd.read_csv("/kaggle/input/trends-assessment-prediction/sample_submission.csv")
 output = sample_submission.drop('Predicted',axis=1 ).merge(output,on='Id',how='left' )<load_from_csv>
day_map = {4:0, 5:1, 3:1, 6:2, 2:2, 7:3, 1:3}
tr['day2'] = tr['day'].map(day_map)
 te['day2'] = te['day'].map(day_map )
sub_df = pd.read_csv(".. /input/trends-svc-rapids-ai/submission.csv" )<create_dataframe>
mm_map = {1:0, 2:0, 3:1, 4:1, 5:2, 6:2, 7:3, 8:3, 9:4, 10:4, 11:5, 12:5}
tr['month2'] =(tr['month'] >= 6 ).astype(int)
 te['month2'] =(te['month'] >= 6 ).astype(int )
final_sub = pd.DataFrame(data={
'id':sample_submission['Id'],
'Predicted': 0.7*output['Predicted'] + 0.3*sub_df['Predicted']
} )<save_to_csv>
bin_cols = ColumnGroup.binary
ord_cols = ColumnGroup.ordinal
nom_cols = ColumnGroup.nominal
cyc_cols = ['day', 'month']
sub_df.to_csv("submission.csv", index=False )<set_options>
pd.set_option('display.max_columns', 200)
 tr.loc[(tr.ord_4 == 11)&(tr.ord_5 == 11)]
!pip install albumentations > /dev/null
!pip install -U efficientnet==0.0.4
plt.style.use('seaborn-white')
 sns.set_style("white")
 seed = 10
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 %matplotlib inline<load_pretrained>
from sklearn.decomposition import TruncatedSVD
import lightgbm as lgb
!mkdir masks
!unzip -q.. /input/data-repack-and-image-statistics/masks.zip -d masks
!mkdir train
!unzip -q.. /input/data-repack-and-image-statistics/train.zip -d train
!mkdir test
!unzip -q.. /input/data-repack-and-image-statistics/test.zip -d test<feature_engineering>
feat_pipe2 = make_union(
 make_pipeline(SelectCols(nom_cols+['month', 'day2']),
LabelEncoderWithThresh(3),
OneHotEncoder() , 
),
make_pipeline(SelectCols(bin_cols)) ,
make_pipeline(SelectCols(ord_cols + ['qtr', 'ord_5a']),
MinMaxScaler(( 0, 1))), 
)
 feat_pipe1 = make_union(
 make_pipeline(SelectCols(nom_cols+['month', 'day']),
LabelEncoderWithThresh(1),
OneHotEncoder() , 
),
make_pipeline(SelectCols(bin_cols)) ,
make_pipeline(SelectCols(ord_cols + ['ord_5a']),
MinMaxScaler(( 0, 1))), 
)
 model = LogisticRegression(C=0.11, solver='lbfgs', random_state=12345786, max_iter=5000, tol=1e-8, fit_intercept=True)
all_mask_fn = glob.glob('./masks/*')
 mask_df = pd.DataFrame()
mask_df['file_names'] = all_mask_fn
mask_df['mask_percentage'] = 0
mask_df.set_index('file_names',inplace=True)
 for fn in all_mask_fn:
mask_df.loc[fn,'mask_percentage'] = np.array(Image.open(fn)).sum() /(256*256*255)
 mask_df.reset_index(inplace=True)
 sns.distplot(mask_df.mask_percentage)
 mask_df['labels'] = 0
mask_df.loc[mask_df.mask_percentage>0,'labels'] = 1<split>
def cross_val_pred(pipe, model, tr, y, te, cvlist):
scores = []
val_preds = []
y_trues = []
test_preds = []
pipe.fit(pd.concat([tr, te]))
for trix, vlix in cvlist:
xtr, ytr = tr.iloc[trix], y[trix]
xvl, yvl = tr.iloc[vlix], y[vlix]
pipe.fit(xtr, ytr)
 xtr = pipe.transform(xtr)
 xvl = pipe.transform(xvl)
 print(xtr.shape)
 model.fit(xtr, ytr)
 pvl = model.predict_proba(xvl)[:, 1]
val_preds.extend(pvl)
 y_trues.extend(yvl)
 score = roc_auc_score(yvl, pvl)
 print("Score ", score)
 scores.append(score)
 print("Avg score ", np.mean(scores), np.std(scores))
print("Overall score ", roc_auc_score(y_trues, val_preds))
xtr = pipe.transform(tr)
 xte = pipe.transform(te)
 test_preds = model.fit(xtr, y ).predict_proba(xte)[:, 1]
return val_preds, y_trues, test_preds
all_train_fn = glob.glob('./train/*')
 total_samples = len(all_train_fn)
 idx = np.arange(total_samples)
 train_fn,val_fn = train_test_split(all_train_fn,stratify=mask_df.labels,test_size=0.1,random_state=10)
 print('No.of train files:', len(train_fn))
print('No.of val files:', len(val_fn))
masks_train_fn = [fn.replace('./train','./masks')for fn in train_fn]
masks_val_fn = [fn.replace('./train','./masks')for fn in val_fn]<define_variables>
val_preds1, y_trues, test_preds1 = cross_val_pred(feat_pipe2, model, tr, y, te, cvlist)
train_im_path,train_mask_path = './keras_im_train','./keras_mask_train'
h,w,batch_size = 256,256,16
val_im_path,val_mask_path = './keras_im_val','./keras_mask_val'
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, train_im_path=train_im_path,train_mask_path=train_mask_path,
augmentations=None, batch_size=batch_size,img_size=256, n_channels=3, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.train_im_paths = glob.glob(train_im_path+'/*')
 self.train_im_path = train_im_path
self.train_mask_path = train_mask_path
self.img_size = img_size
self.n_channels = n_channels
self.shuffle = shuffle
self.augment = augmentations
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.train_im_paths)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:min(( index+1)*self.batch_size,len(self.train_im_paths)) ]
list_IDs_im = [self.train_im_paths[k] for k in indexes]
X, y = self.data_generation(list_IDs_im)
 if self.augment is None:
return X,np.array(y)/255
else:
im,mask = [],[]
for x,y in zip(X,y):
augmented = self.augment(image=x, mask=y)
 im.append(augmented['image'])
 mask.append(augmented['mask'])
 return np.array(im),np.array(mask)/255
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.train_im_paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def data_generation(self, list_IDs_im):
'Generates data containing batch_size samples'
X = np.empty(( len(list_IDs_im),self.img_size,self.img_size, self.n_channels))
y = np.empty(( len(list_IDs_im),self.img_size,self.img_size, 1))
for i, im_path in enumerate(list_IDs_im):
im = np.array(Image.open(im_path))
mask_path = im_path.replace(self.train_im_path,self.train_mask_path)
 mask = np.array(Image.open(mask_path))
if len(im.shape)==2:
im = np.repeat(im[...,None],3,2)
 X[i,] = cv2.resize(im,(self.img_size,self.img_size))
y[i,] = cv2.resize(mask,(self.img_size,self.img_size)) [..., np.newaxis]
y[y>0] = 255
return np.uint8(X),np.uint8(y )<normalization>
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma,OneOf,
ToFloat, ShiftScaleRotate,GridDistortion, ElasticTransform, JpegCompression, HueSaturationValue,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise,CenterCrop,
IAAAdditiveGaussianNoise,GaussNoise,OpticalDistortion,RandomSizedCrop 
)
 AUGMENTATIONS_TRAIN = Compose([
HorizontalFlip(p=0.5),
OneOf([
RandomContrast() ,
RandomGamma() ,
RandomBrightness() ,
], p=0.3),
OneOf([
ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
GridDistortion() ,
OpticalDistortion(distort_limit=2, shift_limit=0.5),
], p=0.3),
RandomSizedCrop(min_max_height=(176, 256), height=h, width=w,p=0.25),
ToFloat(max_value=1)
 ],p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=1)
 ],p=1 )<compute_train_metric>
sub = te[['id']]
sub['target'] = test_preds1
sub.to_csv('submission.csv', index=False )