Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<feature_engineering>
warnings.filterwarnings("ignore")
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = fill_in(market_obs_df)
 market_obs_df = data_other(market_obs_df)
 X_live = market_obs_df[fcol1].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 min_max_scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1))
predictions_template_df[['confidenceValue']] = min_max_scaler.fit_transform(predictions_template_df[['confidenceValue']])
 env.predict(predictions_template_df)
 packaging_time += time.time() - t 
<load_from_csv>
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
Q1 = train.quantile(0.25)
 Q3 = train.quantile(0.75)
 IQR = Q3 - Q1
train = train[~(( train <(Q1 - 1.5 * IQR)) |(train >(Q3 + 1.5 * IQR)) ).any(axis=1)]
import numpy as np
import pandas as pd
import random
from datetime import datetime, date
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import time
import lightgbm as lgb
from kaggle.competitions import twosigmanews<split>
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()<data_type_conversions>
def description(df):
summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.iloc[0].values
summary['Second Value'] = df.iloc[1].values
summary['Third Value'] = df.iloc[2].values
return summary
description(train )
for col in market_train.columns:
if(market_train[col].dtype == "int64" or market_train[col].dtype == "float64"):
market_train[col] = market_train[col].fillna(market_train[col].mean() )<feature_engineering>
bin_dict = {'T':1, 'F':0, 'Y':1, 'N':0}
train['bin_3'] = train['bin_3'].map(bin_dict)
 train['bin_4'] = train['bin_4'].map(bin_dict)
 test['bin_3'] = test['bin_3'].map(bin_dict)
 test['bin_4'] = test['bin_4'].map(bin_dict )
market_train['time'] = market_train.time.dt.date
market_train = market_train.loc[market_train['time'] >= date(2010, 1, 1)]<define_variables>
dummies = pd.concat([train, test], axis=0, sort=False)
 print(f'Shape before dummy transformation: {dummies.shape}')
 dummies = pd.get_dummies(dummies, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\
prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'], drop_first=True)
 print(f'Shape after dummy transformation: {dummies.shape}')
 train, test = dummies.iloc[:train.shape[0], :], dummies.iloc[train.shape[0]:, :]
del dummies
print(train.shape)
 print(test.shape )
asset_code_dict = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
columns_news = ['firstCreated','relevance','sentimentClass','sentimentNegative','sentimentNeutral',
'sentimentPositive','noveltyCount24H','noveltyCount7D','volumeCounts24H','volumeCounts7D','assetCodes','sourceTimestamp',
'assetName','audiences', 'urgency', 'takeSequence', 'bodySize', 'companyCount',
'sentenceCount', 'firstMentionSentence','time']<feature_engineering>
def date_cyc_enc(df, col, max_vals):
df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)
 df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)
 return df
train = date_cyc_enc(train, 'day', 7)
 test = date_cyc_enc(test, 'day', 7)
 train = date_cyc_enc(train, 'month', 12)
 test = date_cyc_enc(test, 'month', 12)
 print(train.shape)
 print(test.shape )
def feature_engineering(df):
df['date'] = df['time']
df['price_diff'] = df['close'] - df['open']
df['close_to_open'] = np.abs(df['close'] / df['open'])
 df['assetName_mean_open'] = df.groupby('assetName')['open'].transform('mean')
 df['assetName_mean_close'] = df.groupby('assetName')['close'].transform('mean')
 df['assetCodeT'] = df['assetCode'].map(asset_code_dict)
 return df
df = feature_engineering(market_train )<data_type_conversions>
train['ord_5_oe_add'] = train['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1)for letter in x]))
test['ord_5_oe_add'] = test['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1)for letter in x]))
train['ord_5_oe_join'] = train['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1)for letter in x)))
 test['ord_5_oe_join'] = test['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1)for letter in x)))
 train['ord_5_oe1'] = train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
test['ord_5_oe1'] = test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
train['ord_5_oe2'] = train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
test['ord_5_oe2'] = test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
for col in ['ord_5_oe1', 'ord_5_oe2', 'ord_5_oe_add', 'ord_5_oe_join']:
train[col]= train[col].astype('float64')
 test[col]= test[col].astype('float64')
 print(train.shape)
 print(test.shape )
def reduce_mem_usage(df):
for col in df.columns:
if df[col].dtype == np.float64 or df[col].dtype == np.float32:
df[col] = df[col].astype(np.float16)
 return df
df = reduce_mem_usage(df )<data_type_conversions>
high_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
for col in high_card_feats:
train[f'hash_{col}'] = train[col].apply(lambda x: hash(str(x)) % 5000)
 test[f'hash_{col}'] = test[col].apply(lambda x: hash(str(x)) % 5000)
 if train[col].dtype == 'object' or test[col].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(train[col].values)+ list(test[col].values))
train[f'le_{col}'] = lbl.transform(list(train[col].values))
test[f'le_{col}'] = lbl.transform(list(test[col].values))
print(train.shape)
 print(test.shape )
for col in df.columns:
if df[col].dtype == "int64" or df[col].dtype == "float16":
df[col] = df[col].fillna(df[col].mean() )<define_variables>
col = high_card_feats + ['ord_5','day', 'month']
train.drop(col, axis=1, inplace=True)
 test.drop(col, axis=1, inplace=True)
 print(train.shape)
 print(test.shape )
up = df['returnsOpenNextMktres10'] >= 0
universe = df['universe'].values
d = df['date']
fcol = [c for c in df if c not in ['date', 'assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
print("Creating X")
 X = df[fcol].values
print("Creating up")
 up = up.values
print("Creating r")
 r = df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
print("Reshaping X")
 X = 1 -(( maxs - X)/ rng)
 print("Rehaped and asseting")
 assert X.shape[0] == up.shape[0] == r.shape[0]
df['time'] = pd.to_datetime(df['date'])
 te = df['time']>date(2015, 1, 1)
 print("Getting index values")
 tt = 0
for tt,i in enumerate(te.values):
if i:
idx = tt
print(i,tt)
 break
print(idx)
 print("Creating X_train, X_test")
 X_train, X_test = X[:idx],X[idx:]
print("Creating up_train, up_test")
 up_train, up_test = up[:idx],up[idx:]
print("Creating r_train, r_test")
 r_train, r_test = r[:idx],r[idx:]
print("Creating u_train, u_test")
 u_train,u_test = universe[:idx],universe[idx:]
print("Creating d_train, d_test")
 d_train,d_test = d[:idx],d[idx:]
print("Creating Train Dataset")
 train_data = lgb.Dataset(X_train, label=up_train.astype(int))
print("Creating Test Dataset")
 test_data = lgb.Dataset(X_test, label=up_test.astype(int))<split>
def get_optimized_column(column):
if not np.issubdtype(column.dtypes, np.number):
return column
integers = [np.int8, np.int16, np.int32, np.int64]
floats = [np.float16, np.float32, np.float64]
max = column.max()
relevant_types = integers if np.issubdtype(column.dtypes, np.integer)else floats
for dtype in relevant_types:
if dtype(max)== max:
return column.astype(dtype)
 return column
def reduce_mem_usage(df, verbose=True):
start_mem = df.memory_usage().sum() / 1024**2
for column in df.columns:
df[column] = get_optimized_column(df[column])
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test)
 description(train )
DATA_SPLIT_SEED = 2018
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=DATA_SPLIT_SEED ).split(X, up))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = X[train_idx]
y_train = up[train_idx]
X_val = X[valid_idx]
y_val = up[valid_idx]
train_data = lgb.Dataset(X_train, label=y_train)
 test_data = lgb.Dataset(X_val, label=y_val)
 x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
def exp_loss(p,y):
y = y.get_label()
grad = -y*(1.0-1.0/(1.0+np.exp(-y*p)))
 hess = -(np.exp(y*p)*(y*p-1)-1)/(( np.exp(y*p)+1)**2)
 return grad,hess
params_1 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': 239,
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': 172,
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5, 
)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5,
)<compute_train_metric>
def runCatBoost(train_pool, validate_pool, params):
print('Train CatBoost model')
 model = CatBoostClassifier(**params)
 model.fit(train_pool, eval_set=validate_pool, plot=True)
 return model
def predict(model, pool):
print('Predict')
 pred_test_y = model.predict_proba(X_test)[:, 1]
return pred_test_y
X_train, X_val, y_train, y_val = train_test_split(
 train, target,
test_size = 0.3,
random_state = 42 
)
 cat_features = ['bin_0','bin_1','bin_2','bin_3','bin_4','ord_0','ord_1','ord_2','ord_3','ord_4']
train_pool = Pool(train, target, cat_features = cat_features)
 validate_pool = Pool(X_val, y_val, cat_features = cat_features)
 test_pool = Pool(data=test, cat_features = cat_features )
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test )<define_variables>
params = {
'iterations': 15000,
'learning_rate': 0.1,
'eval_metric': 'AUC',
'random_seed': 42,
'task_type':'GPU',
'l2_leaf_reg': 10,
'logging_level': 'Silent',
'use_best_model': True,
'custom_loss': ['AUC'],
'metric_period': 50,
'bagging_temperature' : 0.2,
'l2_leaf_reg': 10
}
model1 = runCatBoost(train_pool, validate_pool, params )
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0<normalization>
params.update({
'random_seed': 42,
'learning_rate': 0.05,
'l2_leaf_reg': 5,
'depth': 4
})
def post_scaling(df):
mean, std = np.mean(df), np.std(df)
 df =(df - mean)/(std * 8)
 return np.clip(df,-1,1 )<categorify>
params.update({
'random_seed': 44,
'learning_rate': 0.025,
'l2_leaf_reg': 3,
'depth': 7
})
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 for col in market_obs_df.columns:
if(market_obs_df[col].dtype == "int64" or market_obs_df[col].dtype == "float64"):
market_obs_df[col] = market_obs_df[col].fillna(market_obs_df[col].mean())
 for col in market_obs_df.columns:
if(market_obs_df[col].dtype == "int64" or market_obs_df[col].dtype == "float64"):
market_obs_df[col] = market_obs_df[col].fillna(market_obs_df[col].mean())
 market_obs_df = feature_engineering(market_obs_df)
 fcol = [c for c in market_obs_df if c not in ['date', 'assetCode', 'time', 'returnsOpenNextMktres10', 'universe', 'assetName', 'Unnamed: 0']]
market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
confidence = lp
confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence': post_scaling(confidence)})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df )<load_from_csv>
params.update({
'iterations': 15000,
'random_seed': 45,
'learning_rate': 0.0125,
'l2_leaf_reg': 10,
'depth': 5
})
<import_modules><EOS> 
def submit(model, pool, i):
print('Predict ' + str(i))
results = model.predict_proba(pool)[:, 1]
submission = pd.DataFrame({'id': test_id, 'target': results})
 submission.to_csv('submission' + str(i)+ '.csv', index=False)
 submit(model1, test_pool,1)