Kaggle competition: Categorical Feature Encoding Challenge
dtrain=xgb.DMatrix(X_train,y_train)
 dvalidate=xgb.DMatrix(X_val,y_val)
 param={'max_depth':5,'eta':1,'ojective':'reg:linear'}
model_xg=xgb.train(param,dtrain,200)
 pred_y=model_xg.predict(dvalidate)
 val_y_inv=np.exp(y_val)-1
pred_y_inv=np.exp(pred_y)-1
rmspe_val=rmspe(val_y_inv,pred_y_inv)
 print(rmspe_val )<drop_column>
df_train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 df_test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(df_train.shape)
 print(df_test.shape)
 df_train.info()
test_merged=test_merged.drop(['Id','Customers'],1 )<predict_on_test>
for i in range(10):
col = 'nom_{}'.format(i)
 print(col, df_train[col].nunique())
 for i in range(6):
col = 'ord_{}'.format(i)
 print(col, df_train[col].nunique() )
y_pred_xg=model_xg.predict(xgb.DMatrix(test_merged[X_train.columns]))<prepare_output>
for df in [df_train, df_test]:
for col in cyclic_cols:
df[col+'_sin'] = np.sin(( 2 * np.pi * df[col])/ max(df[col]))
df[col+'_cos'] = np.cos(( 2 * np.pi * df[col])/ max(df[col]))
y_pred_xg_exp=np.exp(y_pred_xg)-1<save_to_csv>
target = df_train['target']
train_id = df_train['id']
test_id = df_test['id']
train0 = df_train.copy().drop(['target', 'id'], axis=1)
 test0 = df_test.copy().drop('id', axis=1)
 print(train0.shape)
 print(test0.shape)
 for df in [train0, test0]:
df['bin_3'] = df['bin_3'].map({'T': 1, 'F': 0})
 df['bin_4'] = df['bin_4'].map({'Y': 1, 'N': 0})
 df[cyclic_cols] /= df[cyclic_cols].max(axis=0)
 df.drop(['ord_{}'.format(i)for i in range(6)], axis=1, inplace=True)
 df[['ord_{}_sort'.format(i)for i in range(6)]] /= df[['ord_{}_sort'.format(i)for i in range(6)]].max(axis=0)
 train0.head()
submission_predicted1 = pd.DataFrame({'Id': test['Id'], 'Sales': y_pred_xg_exp})
 testop0=(test[test['Open']==0]['Open'] ).index
Sales1=[]
for i in range(41088):
if i in testop0:
Sales1.append(0)
 else:
Sales1.append(submission_predicted1['Sales'][i])
 submission_predicted1['Sales']=Sales1
print(submission_predicted1.head())
 submission_predicted1.to_csv('submission.csv', index=False )<set_options>
%%time
col_names = ['nom_{}'.format(i)for i in range(5)] + ['ord_{}'.format(i)for i in range(5)] + ['day', 'month']
train_ohe = df_train[col_names]
test_ohe = df_test[col_names]
traintest = pd.concat([train_ohe, test_ohe])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe1 = dummies.iloc[:df_train.shape[0], :]
test_ohe1 = dummies.iloc[df_train.shape[0]:, :]
print('train_ohe1.shape', train_ohe1.shape)
 col_names = ['nom_{}'.format(i)for i in range(5, 10)] + ['ord_{}'.format(i)for i in range(5, 6)]
train_ohe = df_train[col_names]
test_ohe = df_test[col_names]
traintest = pd.concat([train_ohe, test_ohe])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe2 = dummies.iloc[:df_train.shape[0], :].sparse.to_coo().tocsr()
test_ohe2 = dummies.iloc[df_train.shape[0]:, :].sparse.to_coo().tocsr()
print('train_ohe2.shape', train_ohe2.shape)
 n_components = 200
svd = TruncatedSVD(n_components=n_components)
 train_svd = svd.fit_transform(train_ohe2)
 test_svd = svd.transform(test_ohe2)
 print('train_svd.shape', train_svd.shape)
 train = train0.drop(['nom_{}'.format(i)for i in range(10)], axis=1)
 test = test0.drop(['nom_{}'.format(i)for i in range(10)], axis=1)
 col_names = ['pca_{}'.format(i)for i in range(n_components)]
train = pd.concat([train, train_ohe1, pd.DataFrame(train_svd, columns=col_names)], axis=1)
 test = pd.concat([test, test_ohe1, pd.DataFrame(test_svd, columns=col_names)], axis=1)
 print('train.shape', train.shape )
pd.set_option('display.max_columns', 200)
 %precision 3<load_from_csv>
col_names = ['bin_{}'.format(i)for i in range(5)] \
+ ['nom_{}'.format(i)for i in range(10)] \
+ ['ord_{}'.format(i)for i in range(6)] \
+ ['day', 'month']
train_ohe = df_train[col_names]
test_ohe = df_test[col_names]
traintest = pd.concat([train_ohe, test_ohe])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:df_train.shape[0], :].sparse.to_coo().tocsr()
test_ohe = dummies.iloc[df_train.shape[0]:, :].sparse.to_coo().tocsr()
print('train_ohe.shape', train_ohe.shape )
train = pd.read_csv("/kaggle/input/rossmann-store-sales/train.csv")
 test = pd.read_csv("/kaggle/input/rossmann-store-sales/test.csv")
 store = pd.read_csv("/kaggle/input/rossmann-store-sales/store.csv")
 state = pd.read_csv("/kaggle/input/rossmann-store-extra/store_states.csv")
 state_name = pd.read_csv("/kaggle/input/rossmann-store-extra/state_names.csv")
 weathers = pd.read_csv("/kaggle/input/rossmann-store-extra/weather.csv")
 L = len(test )<feature_engineering>
lr1 = LogisticRegression(solver='lbfgs', C=0.1)
 lr2 = LogisticRegression(solver='lbfgs', C=0.1)
 with ignore_warnings(category=FutureWarning):
%time print('Dense CV:', cross_val_score(lr1, train, target, cv=2, scoring='roc_auc', n_jobs=-1 ).mean())
 %time print('OHE CV:', cross_val_score(lr2, train_ohe, target, cv=2, scoring='roc_auc', n_jobs=-1 ).mean() )
def rmspe(y_pred, y_test):
return np.sqrt(np.mean(((y_pred - y_test)/ y_test)**2))
def date(data):
data['Date'] = pd.to_datetime(data['Date'])
 data['Year'] = data['Date'].dt.year
data['Year'] = data['Year'] - data['Year'].min()
data['Month'] = data['Date'].dt.month - 1
data['Day'] = data['Date'].dt.day - 1
data['DayOfWeek'] = data['Date'].dt.dayofweek
data['WeekOfYear'] = data['Date'].dt.weekofyear
data['Days'] =(data['Date'] - data['Date'].min() ).dt.days
data['Days'] = data['Days'] / data['Days'].max()
data['DayOfMonth'] = [0 if i<=10 else 1 if i<=20 else 2 for i in data['Day']]
data['QuadYear'] = [0 if i<=13 else 1 if i<=26 else 2 if i<=39 else 3 for i in data['WeekOfYear']]
return data
def label(data, column):
unique = data[column].unique()
k = 0
for str in unique:
data.loc[data[column] == str, column] = k
k += 1
def replace(data, weights, features, drop=True):
for feature in features:
data = data.merge(weights[feature], how='left', on=[feature])
 if drop == True:
data = data.drop([feature], axis=1)
 return data
def pre_store(store, state):
store['CompetitionDistance'].fillna(store['CompetitionDistance'].median() , inplace = True)
 store.fillna(0, inplace = True)
 store = store.merge(state, how='left', on=['Store'])
 return store
def pre_weather(data, weather):
weather = weather.rename(columns={'file': 'StateName'})
 weather = weather.merge(state_name, how='left', on=['StateName'])
 weather['Date'] = pd.to_datetime(weather['Date'])
 data = data.merge(weather, how='left', on=['State', 'Date'])
 data['Events'].fillna('NaN', inplace = True)
 label(data, 'Events')
 label(data, 'State')
 data['Max_Gust_SpeedKm_h'].fillna(0, inplace=True)
 data['CloudCover'].fillna(data['CloudCover'].median() , inplace = True)
 data['Max_VisibilityKm'].fillna(data['Max_VisibilityKm'].median() , inplace = True)
 data['Mean_VisibilityKm'].fillna(data['Mean_VisibilityKm'].median() , inplace = True)
 data['Min_VisibilitykM'].fillna(data['Min_VisibilitykM'].median() , inplace = True)
 data['CloudCover'].fillna(data['CloudCover'].median() , inplace = True)
 data = data.drop(['Date', 'StateName'], axis=1)
 return data
def preprocess(train, test, store):
train['Id'] = 0
test.fillna(1, inplace=True)
 data = pd.concat([train, test], sort=False)
 data = date(data)
 data = data[(data['Open']!=0)&(data['Sales']!=0)]
data = pd.merge(data, store, how = 'inner', on = 'Store')
 data['Store'] = data['Store'] - 1
data['CompetitionOpen'] = 12 *(2015 - data.Year - data.CompetitionOpenSinceYear)+ \ 
(data.Month - data.CompetitionOpenSinceMonth)
 data['PromoOpen'] = 12 *(2015 - data.Year - data.Promo2SinceYear)+ \ 
(data.WeekOfYear - data.Promo2SinceWeek)/ 4.0
data['SalesLog'] = data['Sales'].map(math.log)
 data['CustomersLog'] = data['Customers'].map(math.log)
 data.loc[data['StateHoliday'] == 0, 'StateHoliday'] = '0'
label(data, 'StateHoliday')
 label(data, 'StoreType')
 label(data, 'Assortment')
 label(data, 'PromoInterval')
 data = data.drop(['Customers', 'Open'], axis=1)
 return data<choose_model_class>
%%time
lr1.fit(train, target)
 lr2.fit(train_ohe, target )
class EntitiyEmbedding:
def __init__(self):
self.input_model = []
self.output_model = []
self.features = []
self.embeddings = []
def add(self, feature, input_shape, output_shape):
self.features.append(feature)
 self.embeddings.append(feature)
 input_model = Input(shape=(1,), name=(feature + '_input'))
output_model = Embedding(input_shape, output_shape, name=(feature + '_embedding'))(input_model)
 output_model = Reshape(target_shape=(output_shape,))(output_model)
 self.input_model.append(input_model)
 self.output_model.append(output_model)
 def dense(self, feature, output_shape):
self.features.append(feature)
 input_model = Input(shape=(1,), name=(feature + '_input'))
output_model = Dense(output_shape, name=(feature + '_dense'))(input_model)
 self.input_model.append(input_model)
 self.output_model.append(output_model)
 def concatenate(self):
output_model = Concatenate()(self.output_model)
 output_model = Dense(1000, kernel_initializer="uniform" )(output_model)
 output_model = Activation('relu' )(output_model)
 output_model = Dense(500, kernel_initializer="uniform" )(output_model)
 output_model = Activation('relu' )(output_model)
 output_model = Dense(1 )(output_model)
 output_model = Activation('sigmoid' )(output_model)
 self.model = KerasModel(inputs=self.input_model, outputs=output_model)
 self.model.compile(loss='mean_absolute_error', optimizer='adam')
 def split_features(self, X):
X_list = {}
for feature in self.features:
X_list[feature + '_input'] = X[[feature]]
return X_list
def fit(self, X_train, y_train, X_test, y_test, epochs=12, batch_size=128):
self.X_test = X_test
self.model.fit(self.split_features(X_train), y_train,
validation_data=(self.split_features(X_test), y_test),
epochs=epochs,
batch_size=batch_size)
 def predict(self, X=None):
if X is None:
X = self.X_test
pred = self.model.predict(self.split_features(X))
return pred
def get_weight(self):
weights = {}
for feature in self.embeddings:
w = self.model.get_layer(feature + '_embedding' ).get_weights() [0]
columns = []
for i in range(w.shape[1]):
columns.append(feature + '_' + str(i))
w = pd.DataFrame(w, columns=columns)
 w.index.names = [feature]
weights[feature] = w
return weights<categorify>
with ignore_warnings(category=FutureWarning):
lr1_predictions = lr1.predict_proba(test)
 lr1_submission = pd.DataFrame({'id': test_id, 'target': lr1_predictions[:, 1]})
 lr1_submission.to_csv('lr1.csv', index=False)
 lr1_submission.head()
store = pre_store(store, state)
 data = preprocess(train, test, store)
 data = pre_weather(data, weathers)
 smax = data['SalesLog'].max()
cmax = data['CustomersLog'].max()<choose_model_class>
with ignore_warnings(category=FutureWarning):
lr2_predictions = lr2.predict_proba(test_ohe)
 lr2_submission = pd.DataFrame({'id': test_id, 'target': lr2_predictions[:, 1]})
 lr2_submission.to_csv('lr2.csv', index=False)
 lr2_submission.head()
model = EntitiyEmbedding()
model.add('Store', input_shape=1115, output_shape=10)
 model.add('DayOfWeek', input_shape=7, output_shape=6)
 model.add('Year', input_shape=3, output_shape=2)
 model.add('Day', input_shape=31, output_shape=10)
 model.add('Month', input_shape=12, output_shape=6)
 model.add('DayOfMonth', input_shape=3, output_shape=2)
 model.add('QuadYear', input_shape=4, output_shape=3)
 model.add('State', input_shape=12, output_shape=6)
 model.add('StateHoliday', input_shape=4, output_shape=3)
 model.add('WeekOfYear', input_shape=53, output_shape=10)
 model.add('Assortment', input_shape=3, output_shape=2)
 model.add('StoreType', input_shape=4, output_shape=3)
 model.add('PromoInterval', input_shape=4, output_shape=3)
 model.add('Events', input_shape=22, output_shape=10)
 model.dense('Promo', output_shape=1)
 model.concatenate()<split>
%%time
class MyModule(nn.Module):
def __init__(self, num_units=20):
super(MyModule, self ).__init__()
self.dense1 = nn.Linear(train.shape[1], num_units)
 self.dropout = nn.Dropout(0.5)
 self.dense2 = nn.Linear(num_units, 2)
 def forward(self, X, **kwargs):
X = self.dense1(X)
 X = torch.tanh(X)
 X = self.dropout(X)
 X = self.dense2(X)
 X = F.softmax(X, dim=-1)
 return X
net = NeuralNetClassifier(
 MyModule,
max_epochs=20,
lr=0.01,
iterator_train__shuffle=True, 
)
 pipe = Pipeline([ 
('scale', StandardScaler()), 
('net', net),
])
 print('CV:', cross_val_score(pipe,
train.values.astype('float32'),
target.values,
cv=2,
n_jobs=-1,
scoring='roc_auc' ).mean() )
train = data[data['Id']==0]
train = train.sample(frac=1, random_state=22)
 test = data[data['Id']!=0]
Id = test['Id']
33
X = train.copy()
y = train[['CustomersLog', 'SalesLog']]
X_train, X_ee, y_train, y_ee = train_test_split(X, y, test_size=200000, random_state=44 )<train_model>
pipe.fit(train.values.astype('float32'), target.values )
model.fit(X_ee, y_ee['CustomersLog']/cmax, X_train, y_train['CustomersLog']/cmax, epochs=12 )<load_pretrained>
net_predictions = pipe.predict_proba(test.values.astype('float32'))
net_submission = pd.DataFrame({'id': test_id, 'target': net_predictions[:, 1]})
 net_submission.to_csv('net.csv', index=False)
 net_submission.head()
weights = model.get_weight()
X_train = replace(X_train, weights, model.embeddings)
 test = replace(test, weights, model.embeddings)
 X_train = X_train.drop(['Sales', 'Id', 'SalesLog', 'CustomersLog',
'Dew_PointC', 'MeanDew_PointC', 'Min_DewpointC', 'Max_Sea_Level_PressurehPa', 'Mean_Sea_Level_PressurehPa',
'Min_Sea_Level_PressurehPa'], axis=1)
 test = test.drop(['Sales', 'Id', 'SalesLog', 'CustomersLog',
'Dew_PointC', 'MeanDew_PointC', 'Min_DewpointC', 'Max_Sea_Level_PressurehPa', 'Mean_Sea_Level_PressurehPa',
'Min_Sea_Level_PressurehPa'], axis=1)
 X_train.head()<init_hyperparams>
vote_predictions =(lr1_predictions + lr2_predictions)/ 2
vote_submission = pd.DataFrame({'id': test_id, 'target': vote_predictions[:, 1]})
 vote_submission.to_csv('vote1.csv', index=False)
 vote_submission.head()
params = {"objective" : "rmse",
"boosting" : "gbdt",
"metric" : "rmse",
"num_iterations" : 7500,
"top_k" : 30,
"max_depth" : 8,
"num_leaves" : 800,
"min_data_in_leaf" : 20,
"learning_rate" : 0.02,
"bagging_fraction" : 0.7,
"bagging_seed" : 3,
"bagging_freq" : 5,
"feature_fraction" : 0.5,
"num_threads" : 4
}
dataset_params = {"max_bin" : 200,
"min_data_in_bin" : 3
}
lgb_train = lgb.Dataset(X_train, label=y_train['SalesLog'], params=dataset_params)
 model = lgb.train(params, lgb_train, verbose_eval=50, keep_training_booster=True )<save_to_csv>
vote_predictions =(lr1_predictions * 0.1 + lr2_predictions)/ 1.1
vote_submission = pd.DataFrame({'id': test_id, 'target': vote_predictions[:, 1]})
 vote_submission.to_csv('vote2.csv', index=False)
 vote_submission.head()
<load_from_csv><EOS> 
vote_predictions =(lr1_predictions * 0.1 + net_predictions * 0.1 + lr2_predictions)/ 1.2
vote_submission = pd.DataFrame({'id': test_id, 'target': vote_predictions[:, 1]})
 vote_submission.to_csv('vote3.csv', index=False)
 vote_submission.head()