Kaggle competition: Categorical Feature Encoding Challenge
dtrain = xgb.DMatrix(x_train_total, y_train_total)
 dtest = xgb.DMatrix(x_test)
 params = {"objective": "reg:linear",
"booster" : "gbtree",
"eta": 0.03,
"max_depth": 10,
"subsample": 0.9,
"colsample_bytree": 0.7,
"silent": 1,
"seed": 10
}
num_round = 3000
model = xgb.train(params, dtrain, num_round)
 preds = model.predict(dtest )<save_to_csv>
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv', index_col='id')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv', index_col='id' )
result = pd.DataFrame({"Id": test["Id"],'Sales': np.expm1(preds*0.995)})
 result.to_csv("submission_xgb.csv", index=False)
 fig, ax = plt.subplots(figsize=(8,8))
xgb.plot_importance(model, max_num_features= 10, height=0.5, ax=ax)
 plt.show()<import_modules>
train = train.drop('bin_0', axis=1)
 test = test.drop('bin_0', axis=1 )
import pandas as pd
import numpy as np
import cudf<load_from_csv>
map_bin_3_4 = {'T': 1, 'F': 0, 'Y': 1, 'N': 0}
train['bin_3'] = train['bin_3'].map(map_bin_3_4)
 test['bin_3'] = test['bin_3'].map(map_bin_3_4)
 train['bin_4'] = train['bin_4'].map(map_bin_3_4)
 test['bin_4'] = test['bin_4'].map(map_bin_3_4 )
train_scores_df = pd.read_csv("/kaggle/input/trends-assessment-prediction/train_scores.csv", index_col='Id')
 train_scores_df['is_train'] = True
train_scores_df<load_from_csv>
train['ord_5_1'] = train['ord_5'].str[0]
train['ord_5_2'] = train['ord_5'].str[1]
train = train.drop('ord_5', axis=1)
 test['ord_5_1'] = test['ord_5'].str[0]
test['ord_5_2'] = test['ord_5'].str[1]
test = test.drop('ord_5', axis=1 )
loading_df = pd.read_csv("/kaggle/input/trends-assessment-prediction/loading.csv", index_col='Id')
 loading_features = list(loading_df.columns.values)
 fnc_df = pd.read_csv("/kaggle/input/trends-assessment-prediction/fnc.csv", index_col='Id')
 fnc_features = list(fnc_df.columns.values)
 features_df = loading_df.join(fnc_df)
 all_features = list(set(features_df.columns.values)- set(['IC_20']))
features_df.shape<merge>
train = train.drop('ord_5_2', axis=1)
 test = test.drop('ord_5_2', axis=1 )
combined_df = train_scores_df.merge(features_df, how='outer', left_index=True, right_index=True)
 print(combined_df.shape)
 combined_df<count_missing_values>
ord_1_values = ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']
map_ord_1 = lambda x: ord_1_values.index(x)
 train['ord_1'] = train['ord_1'].apply(map_ord_1)
 test['ord_1'] = test['ord_1'].apply(map_ord_1 )
combined_df.is_train.fillna(False, inplace=True)
 combined_df.is_train.mean()<compute_test_metric>
ord_2_values = ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']
map_ord_2 = lambda x: ord_2_values.index(x)
 train['ord_2'] = train['ord_2'].apply(map_ord_2)
 test['ord_2'] = test['ord_2'].apply(map_ord_2 )
def MAPE(true, predicted, **kwargs):
absolute_error = np.abs(predicted-true ).sum()
normalized_error = absolute_error/true.sum()
return normalized_error
MAPE_scorer = make_scorer(MAPE, greater_is_better=False)
 def weighted_absolute_error(true, predicted, weight):
return weight*MAPE(true, predicted )<init_hyperparams>
map_to_ascii_index = lambda x: string.ascii_letters.index(x)
 for column in ['ord_3', 'ord_4', 'ord_5_1']:
train[column] = train[column].apply(map_to_ascii_index)
 test[column] = test[column].apply(map_to_ascii_index )
kf = KFold(n_splits=5, shuffle=True, random_state=0)
 label_weights = {'age':.3,
'domain1_var1':.175,
'domain1_var2':.175,
'domain2_var1':.175,
'domain2_var2':.175}
train_and_val_df = combined_df[combined_df.is_train]
test_df = combined_df[~combined_df.is_train]
def evaluate_model(model_fn, model_name):
fold_scores = []
for fold_i,(train_index, val_index)in enumerate(kf.split(train_and_val_df)) :
weighted_absolute_errors = []
for label in label_weights.keys() :
train_df = train_and_val_df.iloc[train_index]
train_df = train_df[~train_df[label].isna() ]
val_df = train_and_val_df.iloc[val_index]
val_df = val_df[~val_df[label].isna() ]
preprocess, model = model_fn()
preprocess.fit(train_df, train_df[label])
 model.fit(np.asfortranarray(preprocess.transform(train_df)) ,
np.array(train_df[label].values, order='F'))
predicted = np.array(model.predict(np.asfortranarray(preprocess.transform(val_df))))
weighted_absolute_errors.append(weighted_absolute_error(true=val_df[label],
predicted=predicted,
weight=label_weights[label]))
score = sum(weighted_absolute_errors)
 print("[%s] Fold %d, score: %f"%(model_name, fold_i, score))
fold_scores.append(score)
 print("[%s] Average score: %f"%(model_name, np.mean(fold_scores)))
 def create_dummy_model() :
model = DummyRegressor('median')
 preprocess = Pipeline([ 
('union', ColumnTransformer([('scale', RobustScaler() , loading_features)]))
])
 return preprocess, model
evaluate_model(create_dummy_model, 'median')
 def create_linear_regression_model() :
regressor = LinearRegression()
pipeline = Pipeline([ 
('union', ColumnTransformer([('scale', RobustScaler() , loading_features)])) , 
("dummy_regressor", regressor)
 ])
 return pipeline
def create_gb_regression_model() :
regressor = GradientBoostingRegressor()
pipeline = Pipeline([ 
('union', ColumnTransformer([('scale', RobustScaler() , loading_features)])) , 
("dummy_regressor", regressor)
 ])
 return pipeline
def create_sv_regression_model() :
regressor = SVR()
pipeline = Pipeline([ 
('union', ColumnTransformer([('scale', RobustScaler() , loading_features)])) , 
("dummy_regressor", regressor)
 ])
 return pipeline
def create_cusvr_model() :
model = cuSVR(cache_size=3000.0)
 preprocess = Pipeline([ 
('union', ColumnTransformer([('scale', RobustScaler() , loading_features)]))
])
 return preprocess, model
evaluate_model(create_cusvr_model, 'cuSVR' )<normalization>
columns_to_test = ['nom_7', 'nom_8', 'nom_9']
replace_xor = lambda x: 'xor' if x in xor_values else x
for column in columns_to_test:
xor_values = set(train[column].unique())^ set(test[column].unique())
 if xor_values:
print('Column', column, 'has', len(xor_values), 'XOR values')
 train[column] = train[column].apply(replace_xor)
 test[column] = test[column].apply(replace_xor)
 else:
print('Column', column, 'has no XOR values' )
warnings.simplefilter("ignore")
 def model_factory(C, factor):
def create_svr_model_v2() :
def down_scale(x):
return x/factor
PREPROCESSING_STAGE_NAME = 'preprocessing'
REGRESSION_STAGE_NAME = 'regression'
preprocess = Pipeline([ 
(PREPROCESSING_STAGE_NAME,
ColumnTransformer([('down_scale_fnc', FunctionTransformer(down_scale), fnc_features), 
('scale_others', 'passthrough', loading_features)])) ,
])
 model = cuSVR(C=C, cache_size=3000.0)
 return preprocess, model
return create_svr_model_v2
for C in [1, 10, 100]:
for factor in [200.0, 250.0, 500.0]:
evaluate_model(model_factory(C, factor), "C=%f, factor=%f"%(C,factor))<set_options>
def get_infrequent_values(data, column, threshold):
value_counts = data[column].value_counts()
return list(value_counts[value_counts < threshold].index )
warnings.simplefilter("ignore")
 def model_factory_v3(C, k):
def create_svr_model_v3() :
PREPROCESSING_STAGE_NAME = 'preprocessing'
preprocess = Pipeline([ 
(PREPROCESSING_STAGE_NAME,
ColumnTransformer([('top_k', SelectKBest(f_regression, k=k), fnc_features), 
('others', 'passthrough', loading_features)])) , 
('scaling', StandardScaler())
 ])
 model = cuSVR(C=C, cache_size=3000.0)
 return preprocess, model
return create_svr_model_v3
for C in [1, 10, 100]:
for k in [10, 100, 200]:
evaluate_model(model_factory_v3(C, k), "C=%f, k=%f"%(C,k))<set_options>
for column in train.columns:
n = len(get_infrequent_values(train, column, 3))
if n > 0:
print('Column', column, 'has', n, 'unique infrequent value(s)' )
warnings.simplefilter("ignore")
 def model_factory_v4(C, k):
def create_svr_model_v4() :
PREPROCESSING_STAGE_NAME = 'preprocessing'
preprocess = Pipeline([ 
(PREPROCESSING_STAGE_NAME,
ColumnTransformer([('top_k', PCA(n_components=k), fnc_features), 
('others', 'passthrough', loading_features)])) , 
('scaling', StandardScaler())
 ])
 model = cuSVR(C=C, cache_size=3000.0)
 return preprocess, model
return create_svr_model_v4
for C in [1, 10, 100]:
for k in [10, 100, 200]:
evaluate_model(model_factory_v4(C, k), "C=%f, k=%f"%(C,k))<normalization>
thresholds = {
'nom_9': 3
}
for column in thresholds.keys() :
values_to_replace = get_infrequent_values(train, column, thresholds[column])
 replace = lambda x: 'value' if x in values_to_replace else x
train[column] = train[column].transform(replace)
 test[column] = test[column].transform(replace )
warnings.simplefilter("ignore")
 def model_factory_v5(C, k, factor):
def create_svr_model_v5() :
def down_scale(x):
return x/factor
PREPROCESSING_STAGE_NAME = 'preprocessing'
preprocess = Pipeline([ 
(PREPROCESSING_STAGE_NAME,
ColumnTransformer([('top_k', Pipeline([('PCA', PCA(n_components=k)) , 
('scale', FunctionTransformer(down_scale)) ]), fnc_features), 
('others', 'passthrough', loading_features)]))
])
 model = cuSVR(C=C, cache_size=3000.0)
 return preprocess, model
return create_svr_model_v5
for C in [10, 20]:
for k in [200, 300, 400]:
for factor in [250]:
evaluate_model(model_factory_v5(C, k, factor), "C=%f, k=%f, factor=%f"%(C,k, factor))<find_best_params>
for column in thresholds.keys() :
print(column, ':', train[column].value_counts() ['value'], 'values replaced in train dataset' )
for C in [5, 10, 15]:
for k in [500, 600, 700]:
for factor in [250, 350]:
evaluate_model(model_factory_v5(C, k, factor), "C=%f, k=%f, factor=%f"%(C,k, factor))<find_best_params>
all_features = test.columns
selected_features_for_thermometer = ['ord_1', 'ord_4', 'day', 'month', 'ord_5_1']
selected_features_for_both = ['ord_2', 'ord_3']
for C in [15]:
for k in [800, 900]:
for factor in [450, 550]:
evaluate_model(model_factory_v5(C, k, factor), "C=%f, k=%f, factor=%f"%(C,k, factor))<train_model>
y_train = train['target'].copy()
x_train = train.drop('target', axis=1)
 del train
x_test = test.copy()
del test
output_dfs = []
for label in label_weights.keys() :
train_df = train_and_val_df
train_df = train_df[~train_df[label].isna() ]
preprocess, model = model_factory_v5(C=10.000000, k=700, factor=250.0 )()
preprocess.fit(train_df, train_df[label])
 model.fit(np.asfortranarray(preprocess.transform(train_df)) ,
np.array(train_df[label].values, order='F'))
predicted = np.array(model.predict(np.asfortranarray(preprocess.transform(test_df))))
output_dfs.append(pd.DataFrame(predicted, index=test_df.index, columns=[label]))
output_df = pd.concat(output_dfs, axis=1 )<feature_engineering>
class ThermometerEncoder(BaseEstimator, TransformerMixin):
def __init__(self, cols=None, drop_invariant=True):
self.cols = cols
self.drop_invariant = drop_invariant
def get_params(self, deep=True):
return {'cols': self.cols, 'drop_invariant': self.drop_invariant}
def set_params(self, **parameters):
for parameter, value in parameters.items() :
setattr(self, parameter, value)
 return self
def fit(self, X, y=None):
self.bars = {}
for c in self.cols:
k = np.arange(X[c].max() + 1)
 self.bars[c] =(k[:-1] < k.reshape(-1, 1)).astype(int)
 return self
def transform(self, X, y=None):
out = pd.DataFrame(index=X.index)
 for c in self.cols:
out = out.join(self.transform_one(X, c))
if self.drop_invariant:
columns_to_drop = []
for c in out.columns:
if len(out[c].unique())== 1:
columns_to_drop.append(c)
 out = out.drop(columns_to_drop, axis=1)
 return out
def transform_one(self, X, c):
bars = self.bars[c]
out = pd.DataFrame(index=X.index, data=bars[X[c]])
 out.columns = [c + '_' + str(k)for k in range(bars.shape[1])]
return out
melt_df = output_df.reset_index().melt(id_vars=['Id'],
value_vars=['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2'],
value_name='Predicted')
 melt_df['Id'] = melt_df['Id'].astype('str')+ '_' + melt_df['variable'] 
<save_to_csv>
train_part = len(x_train)
 traintest = pd.concat([x_train, x_test] )
melt_df.to_csv("submission.csv", index=False, columns=['Id', 'Predicted'] )<install_modules>
ohe_features = set(all_features)- set(selected_features_for_thermometer )
!pip install -q joypy<import_modules>
ohe_traintest = pd.get_dummies(traintest[ohe_features], columns=ohe_features, drop_first=True, sparse=True ).sparse.to_coo().tocsr()
from sklearn.model_selection import KFold, train_test_split
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import joypy
from tqdm.notebook import tqdm
from glob import glob
import gc
import nilearn as nl
import nilearn.plotting as nlplt
import nibabel as nib
import h5py
import lightgbm as lgb
from scipy.stats import skew, kurtosis
import os, random
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.utils import Sequence<load_from_csv>
thermometer = ThermometerEncoder(selected_features_for_thermometer + selected_features_for_both)
 thermometer_traintest = thermometer.fit_transform(traintest )
MAIN_DATA_PATH = '/kaggle/input/trends-assessment-prediction/'
train_scores_df = pd.read_csv(MAIN_DATA_PATH + 'train_scores.csv')
 icn_numbers_df = pd.read_csv(MAIN_DATA_PATH + 'ICN_numbers.csv')
 loading_df = pd.read_csv(MAIN_DATA_PATH + 'loading.csv')
 fnc_df = pd.read_csv(MAIN_DATA_PATH + 'fnc.csv' )<find_best_params>
final_traintest = scipy.sparse.hstack([ohe_traintest, thermometer_traintest] ).tocsr()
print("Kurtosis(Fisher's definition)")
 train_scores_df.kurtosis()<count_missing_values>
final_x_train = final_traintest[:train_part]
final_x_test = final_traintest[train_part:]
round(train_scores_df.isna().sum() / len(train_scores_df)* 100, 2 )<correct_missing_values>
del x_train
del x_test
train_scores_df.fillna(train_scores_df.mean() , inplace=True )<count_missing_values>
from sklearn.model_selection import KFold
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score
from cross_validation_framework import *
train_scores_df.isna().sum()<merge>
logit_param_grid = {
'C': [0.100, 0.150, 0.120, 0.125, 0.130, 0.135, 0.140, 0.145, 0.150]
}
logit_grid = GridSearchCV(LogisticRegression(solver='lbfgs'), logit_param_grid,
scoring='roc_auc', cv=5, n_jobs=-1, verbose=0)
 logit_grid.fit(final_x_train, y_train)
 best_C = logit_grid.best_params_['C']
print('Best C:', best_C )
features_df = pd.merge(train_scores_df, loading_df, on=['Id'], how='left')
 features_df.head()<count_missing_values>
logit = LogisticRegression(C=best_C, solver='lbfgs', max_iter=10000)
 cv = KFold(n_splits=10, random_state=42)
 _, trained_estimators = fit(ScikitLearnPredictProbaEstimator(logit), roc_auc_score, final_x_train, y_train, cv)
 y = predict(trained_estimators, final_x_test )
fnc_df.isna().sum().sum()<merge>
logit = LogisticRegression(C=0.123456789, solver="lbfgs", max_iter=5000)
 logit.fit(final_x_train, y_train)
 y_full_train = logit.predict_proba(final_x_test)[:, 1]
features_df = pd.merge(features_df, fnc_df, how='left', on='Id')
 features_df.head()<load_from_url>
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id')
 submission['target'] = y
submission.to_csv('logit.csv' )
<categorify><EOS> 
submission['target'] = y_full_train
submission.to_csv('logit-full-train.csv' )