Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<load_from_csv>
import numpy as np
import pandas as pd
import scipy
import sklearn
test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )<predict_on_test>
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
preds=[]
for i in test['text']:
preds.append(get_prediction(i))<load_from_csv>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_validate
sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )<prepare_output>
df_train = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv', index_col='id')
 df_test = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv', index_col='id' )
sub['target']=preds<save_to_csv>
sample_submission = pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv', index_col='id' )
sub.to_csv('submission.csv',index=False )<import_modules>
binvar = ['bin_' + str(i)for i in range(1,5)]
ordvar = ['ord_' + str(i)for i in range(6)]
nomvar = ['nom_' + str(i)for i in range(10)]
dmvar = ['day', 'month']
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
import gc
from sklearn.linear_model import LinearRegression
import random
import datetime as dt
import matplotlib.pyplot as plt
import seaborn as sb 
<set_options>
df_work = df_train.copy()
y_train = df_work['target'].copy()
df_work = pd.concat([df_work.drop('target', axis = 1), df_test] )
%matplotlib inline
sb.set_style("whitegrid")
 ints = ['parcelid']
floats = ['basementsqft', 'bathroomcnt', 'bedroomcnt', 'calculatedbathnbr', 'finishedfloor1squarefeet',
'calculatedfinishedsquarefeet', 'finishedsquarefeet12', 'finishedsquarefeet13',
'finishedsquarefeet15', 'finishedsquarefeet50', 'finishedsquarefeet6', 'fireplacecnt',
'fullbathcnt', 'garagecarcnt', 'garagetotalsqft', 'latitude', 'longitude',
'lotsizesquarefeet', 'poolcnt', 'poolsizesum', 'roomcnt', 'threequarterbathnbr', 'unitcnt',
'yardbuildingsqft17', 'yardbuildingsqft26', 'yearbuilt', 'numberofstories',
'structuretaxvaluedollarcnt', 'taxvaluedollarcnt', 'assessmentyear',
'landtaxvaluedollarcnt', 'taxamount', 'taxdelinquencyyear']
objects = ['airconditioningtypeid', 'architecturalstyletypeid', 'buildingclasstypeid',
'buildingqualitytypeid', 'decktypeid', 'fips', 'hashottuborspa', 'heatingorsystemtypeid',
'pooltypeid10', 'pooltypeid2', 'pooltypeid7', 'propertycountylandusecode',
'propertylandusetypeid', 'propertyzoningdesc', 'rawcensustractandblock', 'regionidcity',
'regionidcounty', 'regionidneighborhood', 'regionidzip', 'storytypeid',
'typeconstructiontypeid', 'fireplaceflag', 'taxdelinquencyflag', 'censustractandblock']
feature_dtypes = {col: col_type for type_list, col_type in zip([ints, floats, objects],
['int64', 'float64', 'object'])
 for col in type_list}
data = pd.read_csv('.. /input/properties_2016.csv' , dtype = feature_dtypes)
 data.head() 
<define_variables>
df_work.drop('bin_0', inplace=True, axis=1 )
continuous = ['basementsqft', 'finishedfloor1squarefeet', 'calculatedfinishedsquarefeet',
'finishedsquarefeet12', 'finishedsquarefeet13', 'finishedsquarefeet15',
'finishedsquarefeet50', 'finishedsquarefeet6', 'garagetotalsqft', 'latitude',
'longitude', 'lotsizesquarefeet', 'poolsizesum', 'yardbuildingsqft17',
'yardbuildingsqft26', 'yearbuilt', 'structuretaxvaluedollarcnt', 'taxvaluedollarcnt',
'landtaxvaluedollarcnt', 'taxamount']
discrete = ['bathroomcnt', 'bedroomcnt', 'calculatedbathnbr', 'fireplacecnt', 'fullbathcnt',
'garagecarcnt', 'poolcnt', 'roomcnt', 'threequarterbathnbr', 'unitcnt',
'numberofstories', 'assessmentyear', 'taxdelinquencyyear']
print("
Continuous variable plots
")
 for col in continuous:
values = data[col].dropna()
lower = np.percentile(values, 1)
 upper = np.percentile(values, 99)
 fig = plt.figure(figsize=(18,9)) ;
sb.distplot(values[(values>lower)&(values<upper)], color='purple', ax = plt.subplot(121)) ;
plt.suptitle(col, fontsize=16 )<load_from_csv>
df_work['bin_3'] = df_work['bin_3'].map({'F':0, 'T':1})
 df_work['bin_4'] = df_work['bin_4'].map({'N':0, 'Y':1} )
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv")
 XGB_WEIGHT = 0.6415
BASELINE_WEIGHT = 0.0050
XGB1_WEIGHT = 0.8083
BASELINE_PRED = 0.0115
print("
Re-reading properties file...")
 properties = pd.read_csv('.. /input/properties_2016.csv')
 print("
Processing data for XGBoost...")
 for c in properties.columns:
properties[c]=properties[c].fillna(-1)
 if properties[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(properties[c].values))
properties[c] = lbl.transform(list(properties[c].values))
train_df = train.merge(properties, how='left', on='parcelid')
 train_df["transactiondate"] = pd.to_datetime(train_df["transactiondate"])
 train_df["Month"] = train_df["transactiondate"].dt.month
x_train = train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 x_test = properties.drop(['parcelid'], axis=1)
 x_test["transactiondate"] = '2016-07-01'
x_test["transactiondate"] = pd.to_datetime(x_test["transactiondate"])
 x_test["Month"] = x_test["transactiondate"].dt.month
x_test = x_test.drop(['transactiondate'], axis=1)
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))
train_df=train_df[ train_df.logerror > -0.4 ]
train_df=train_df[ train_df.logerror < 0.419 ]
x_train=train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 y_train = train_df["logerror"].values.astype(np.float32)
 y_mean = np.mean(y_train)
 print('After removing outliers:')
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.037,
'max_depth': 5,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'lambda': 0.8,
'alpha': 0.4,
'base_score': y_mean,
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
 num_boost_rounds = 250
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost...")
 xgb_pred1 = model.predict(dtest)
 print("
First XGBoost predictions:")
 print(pd.DataFrame(xgb_pred1 ).head())
 print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.033,
'max_depth': 6,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'base_score': y_mean,
'silent': 1
}
num_boost_rounds = 150
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost again...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost again...")
 xgb_pred2 = model.predict(dtest)
 print("
Second XGBoost predictions:")
 print(pd.DataFrame(xgb_pred2 ).head())
 xgb_pred = XGB1_WEIGHT*xgb_pred1 +(1-XGB1_WEIGHT)*xgb_pred2
print("
Combined XGBoost predictions:")
 print(pd.DataFrame(xgb_pred ).head())
 del train_df
del x_train
del x_test
del properties
del dtest
del dtrain
del xgb_pred1
del xgb_pred2
gc.collect()<load_from_csv>
df_work['ord_0'] = df_work['ord_0'] - 1
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv", parse_dates=["transactiondate"])
 properties = pd.read_csv(".. /input/properties_2016.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 OLS_WEIGHT = 0.0856
np.random.seed(17)
 random.seed(17)
 print(len(train),len(properties),len(submission))
def get_features(df):
df["transactiondate"] = pd.to_datetime(df["transactiondate"])
 df["transactiondate_year"] = df["transactiondate"].dt.year
df["transactiondate_month"] = df["transactiondate"].dt.month
df['transactiondate'] = df['transactiondate'].dt.quarter
df = df.fillna(-1.0)
 return df
def MAE(y, ypred):
return np.sum([abs(y[i]-ypred[i])for i in range(len(y)) ])/ len(y)
 train = pd.merge(train, properties, how='left', on='parcelid')
 y = train['logerror'].values
test = pd.merge(submission, properties, how='left', left_on='ParcelId', right_on='parcelid')
 properties = []
exc = [train.columns[c] for c in range(len(train.columns)) if train.dtypes[c] == 'O'] + ['logerror','parcelid']
col = [c for c in train.columns if c not in exc]
train = get_features(train[col])
 test['transactiondate'] = '2016-01-01'
test = get_features(test[col])
 reg = LinearRegression(n_jobs=-1)
 reg.fit(train, y); print('fit...')
 print(MAE(y, reg.predict(train)))
 train = []; y = []
test_dates = ['2016-10-01','2016-11-01','2016-12-01','2017-10-01','2017-11-01','2017-12-01']
test_columns = ['201610','201611','201612','201710','201711','201712'] 
<load_from_csv>
ord1dict = {'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4}
df_work['ord_1'] = df_work['ord_1'].map(ord1dict )
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv")
 XGB_WEIGHT = 0.6415
BASELINE_WEIGHT = 0.0050
XGB1_WEIGHT = 0.8083
BASELINE_PRED = 0.0115
print("
Processing data for LightGBM...")
 for c, dtype in zip(prop.columns, prop.dtypes):	
if dtype == np.float64:		
prop[c] = prop[c].astype(np.float32)
 df_train = train.merge(prop, how='left', on='parcelid')
 df_train.fillna(df_train.median() ,inplace = True)
 df_train["transactiondate"] = pd.to_datetime(df_train["transactiondate"])
 df_train["Month"] = df_train["transactiondate"].dt.month
x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', 'fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train['logerror'].values
print(x_train.shape, y_train.shape)
 train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True)
 del df_train; gc.collect()
x_train = x_train.values.astype(np.float32, copy=False)
 d_train = lgb.Dataset(x_train, label=y_train)
 params = {}
params['max_bin'] = 10
params['learning_rate'] = 0.0021
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'l1'
params['sub_feature'] = 0.345
params['bagging_fraction'] = 0.85
params['bagging_freq'] = 40
params['num_leaves'] = 512
params['min_data'] = 500
params['min_hessian'] = 0.05
params['verbose'] = 0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
np.random.seed(0)
 random.seed(0)
 print("
Fitting LightGBM model...")
 clf = lgb.train(params, d_train, 430)
 del d_train; gc.collect()
del x_train; gc.collect()
print("
Prepare for LightGBM prediction...")
 print(" Read sample file...")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 print("...")
 sample['parcelid'] = sample['ParcelId']
print(" Merge with property data...")
 df_test = sample.merge(prop, on='parcelid', how='left')
 df_test["transactiondate"] = '2016-07-01'
df_test["transactiondate"] = pd.to_datetime(df_test["transactiondate"])
 df_test["Month"] = df_test["transactiondate"].dt.month
df_test = df_test.drop(['transactiondate'], axis=1)
 print("...")
 del sample, prop; gc.collect()
print("...")
 x_test = df_test[train_columns]
print("...")
 del df_test; gc.collect()
print(" Preparing x_test...")
 for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] =(x_test[c] == True)
 print("...")
 x_test = x_test.values.astype(np.float32, copy=False)
 print("
Start LightGBM prediction...")
 p_test = clf.predict(x_test)
 del x_test; gc.collect()
print("
Unadjusted LightGBM predictions:")
 print(pd.DataFrame(p_test ).head())
<prepare_output>
ord2dict = {'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
df_work['ord_2'] = df_work['ord_2'].map(ord2dict )
print("
Combining XGBoost, LightGBM, and baseline predicitons...")
 lgb_weight =(1 - XGB_WEIGHT - BASELINE_WEIGHT)/(1 - OLS_WEIGHT)
 xgb_weight0 = XGB_WEIGHT /(1 - OLS_WEIGHT)
 baseline_weight0 = BASELINE_WEIGHT /(1 - OLS_WEIGHT)
 pred0 = xgb_weight0*xgb_pred + baseline_weight0*BASELINE_PRED + lgb_weight*p_test
print("
Combined XGB/LGB/baseline predictions:")
 print(pd.DataFrame(pred0 ).head())
 print("
Predicting with OLS and combining with XGB/LGB/baseline predicitons:...")
 for i in range(len(test_dates)) :
test['transactiondate'] = test_dates[i]
pred = OLS_WEIGHT*reg.predict(get_features(test)) +(1-OLS_WEIGHT)*pred0
submission[test_columns[i]] = [float(format(x, '.4f')) for x in pred]
print('predict...', i)
 print("
Combined XGB/LGB/baseline/OLS predictions:")
 print(submission.head() )<save_to_csv>
oe = OrdinalEncoder(categories='auto')
 df_work[ordvar[3:]] = oe.fit_transform(df_work[ordvar[3:]])
 for var, cl in zip(ordvar[3:], oe.categories_):
print(var)
 print(cl )
print("
Writing results to disk...")
 submission.to_csv('TEm07sub{}.csv'.format(datetime.now().strftime('%Y%m%d_%H%M%S')) , index=False)
<define_variables>
df_work[ordvar] = StandardScaler().fit_transform(df_work[ordvar] )
ACTUALLY_DO_SOMETHING = True
CORRECT_FOR_BUG = True<import_modules>
df_work[nomvar[5:]].nunique()
from tqdm import tqdm<import_modules>
df_work['nom_5'] = df_work['nom_5'].str[4:]
df_work['nom_6'] = df_work['nom_6'].str[3:]
df_work['nom_7'] = df_work['nom_7'].str[3:]
df_work['nom_8'] = df_work['nom_8'].str[3:]
df_work['nom_9'] = df_work['nom_9'].str[3:]
import gzip<train_model>
df_work[nomvar[5:]].nunique()
ff = 1.08
weights = [ 0.27, 0.18,
0.20, 0.35 ] 
<count_values>
enc = OneHotEncoder(categories = 'auto', dtype = 'float64', drop = 'first')
 nom_matrix = enc.fit_transform(df_work[nomvar])
 df_work.drop(nomvar, inplace=True, axis=1 )
print(sum(weights))<feature_engineering>
enc = OneHotEncoder(categories='auto', dtype = 'float64', drop = 'first')
 dm_matrix = enc.fit_transform(df_work[dmvar])
 df_work.drop(dmvar, inplace=True, axis=1 )
if CORRECT_FOR_BUG:
weights[1] /= 10<define_variables>
df_work_sprs =scipy.sparse.hstack([nom_matrix,
scipy.sparse.coo_matrix(df_work ).astype('float64'),
dm_matrix] ).tocsr()
display(df_work_sprs )
fnames =('output_dataset2017a.csv',
'finalXgbApril20171017_011204.csv',
'2017Q4.csv',
'catboost_opt4.csv')
 output="withReoptimizedXGB.csv"<save_to_csv>
X_train = df_work_sprs[:y_train.shape[0]]
X_test = df_work_sprs[y_train.shape[0]:]
if ACTUALLY_DO_SOMETHING:
for id, r in zip(ids,result):
fs.write(id)
 for j in range(6):
fs.write("," +str(round(r[j],6)))
 fs.write("
" )<set_options>
%%time
C = 0.12
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 score = cross_validate(clf, X_train, y_train, cv=3, scoring="roc_auc")
 mean = score['test_score'].mean()
print(score['test_score'])
 print('C =', C, f'{mean:.8f}' )
if ACTUALLY_DO_SOMETHING:
fs.close()<load_pretrained>
%%time
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 clf.fit(X_train, y_train )
if ACTUALLY_DO_SOMETHING:
f_in = open(output, 'rb')
 f_out = gzip.open(output+'.gz', 'wb')
 f_out.writelines(f_in)
 f_out.close()
f_in.close()<train_model>
y_preds = clf.predict_proba(X_test)[:,1]
2+2<define_variables>
sample_submission['target'] = y_preds
sample_submission.to_csv('submission.csv' )