Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.misc import toimage
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.constraints import maxnorm
from keras.optimizers import SGD
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
class Best_clf_cv_transformer(BaseEstimator, TransformerMixin):
def __init__(self, myparams={'name':'LSvc', 'C':1}, **other_params):
self.myparams = myparams
self.myinit(**other_params)
 return
def myinit(self, **other_params):
self.cv = 5
if 'cv' in self.myparams:
self.cv= self.myparams['cv']
clf = None
name = self.myparams['name']
if name == 'Logit':
clf = LogisticRegression(random_state=0)
 elif name == 'DT':
clf = DecisionTreeClassifier(random_state=0)
 elif name == 'RidgClf':
clf = RidgeClassifier(random_state=0)
 elif name == 'Prcpt':
clf = Perceptron(random_state=0)
 elif name == 'PssAggClf':
clf = PassiveAggressiveClassifier(random_state=0)
 elif name == 'Knn':
clf = KNeighborsClassifier(random_state=0)
 elif name == 'RF':
clf = RandomForestClassifier(random_state=0)
 elif name == 'NearCent':
clf = NearestCentroid(random_state=0)
 elif name == 'MultNB':
clf = MultinomialNB(random_state=0)
 elif name == 'BernNB':
clf = BernoulliNB(random_state=0)
 elif name == 'Svc':
clf = SVC(probability=True, random_state=0)
 elif name == 'LSvc':
clf = LinearSVC(random_state=0)
 elif name == 'Xgb':
clf = xgb.XGBClassifier(random_state=0)
 elif name == 'Catb' :
clf = CatBoostClassifier(verbose=False, random_state=0)
 elif name == 'FCNN':
clf = None
else:
print('ERROR Best_clf_cv_transformer: invalid @param name
')
 clf = None
self.isCV = True
if 'isCV' in self.myparams:
self.isCV = self.myparams['isCV']
self.n_estimators = 1
if 'n_estimators' in self.myparams:
self.n_estimators = self.myparams['n_estimators']
if 'params' in self.myparams:
clf.set_params(**self.myparams['params'])
 if other_params:
clf.set_params(**other_params)
 self.param_grid = None
if 'param_grid' in self.myparams:
self.param_grid = self.myparams['param_grid']
self.clf = clf
self.cv_score = 0
self.name = name
self._estimator_type='classifier'
return
def fit(self, X, Y, **FIT_PARAMS):
train_len = len(Y[pd.isnull(Y)==False])
 X, Y = X[:train_len], Y[:train_len]
print('training', self.name, 'for X.shape =', X.shape)
 n_jobs = -1
if self.name == 'FCNN':
lrScheduler = LRScheduler(
 CyclicLR,
base_lr=0.0001,
max_lr=0.05,
step_every='batch' 
)
 self.clf = NeuralNetClassifier(
 module=MyModule,
module__inputCount=X.shape[1],
module__outputCount=2,
module__hiddenLayerCounts=[15],
max_epochs=1000,
verbose=0,
iterator_train__shuffle=True,
callbacks=[('LRScheduler', lrScheduler),('EarlyStopping', EarlyStopping(patience=20)) ] 
)
 X = X.astype(np.float32)
 Y = Y.astype(np.int64)
 n_jobs = None
if self.isCV:
if self.param_grid:
gridSearchCV = GridSearchCV(
 self.clf, self.param_grid, iid=False, cv=self.cv, scoring=CV_SCORERS,
refit='f1_score', n_jobs=n_jobs 
)
 gridSearchCV.fit(X, Y)
 print(self.name, ": Best_clf_cv_transformer: Best parameter(CV score=%0.3f):" % gridSearchCV.best_score_)
 print(gridSearchCV.best_params_)
 self.clf = gridSearchCV.best_estimator_
self.cv_score = gridSearchCV.best_score_
if self.name == 'LSvc':
self.clf = CalibratedClassifierCV(self.clf)
 else:
if self.name == 'LSvc':
self.clf = CalibratedClassifierCV(self.clf)
 print(self.name, ': Best_clf_cv_transformer: starting CV =', self.cv)
 if self.name not in {'fss'}:
cv_results = cross_validate(self.clf, X, Y, cv=int(self.cv))
else:
voting_clf = VotingClassifier(estimators=[(self.name, self.clf)])
 cv_results = cross_validate(voting_clf, X, Y, cv=self.cv)
 self.cv_score = np.mean(cv_results['test_score'])
 print(self.name, ": cv_score: %0.3f" % self.cv_score)
 else:
if self.name == 'LSvc':
self.clf = CalibratedClassifierCV(self.clf)
 if self.n_estimators > 1:
self.clf = BaggingClassifier(base_estimator=self.clf, n_estimators=self.n_estimators)
 self.clf.fit(X, Y)
 print("Done Fitting", self.name)
 return self
def get_cv_score(self):
return self.cv_score
def transform(self, X, Y=None, **FIT_PARAMS):
if self.name == 'FCNN':
X = X.astype(np.float32)
 if not Y is None:
Y = Y.astype(np.int64)
 return self.clf.transform(X, Y)
 def predict(self, X, **FIT_PARAMS):
if self.name == 'FCNN':
X = X.astype(np.float32)
 return self.clf.predict(X)
 def predict_proba(self, X):
if self.name == 'FCNN':
X = X.astype(np.float32)
 return self.clf.predict_proba(X)
 def predict_log_proba(self, X):
if self.name == 'FCNN':
X = X.astype(np.float32)
 return self.clf.predict_log_proba(X)
 def score(self, X, Y, **FIT_PARAMS):
if self.name == 'FCNN':
X = X.astype(np.float32)
 return self.clf.score(X, Y, **FIT_PARAMS)
 def decision_function(self, X, **FIT_PARAMS):
if self.name == 'FCNN':
X = X.astype(np.float32)
 return self.clf.decision_function(X)
 def set_params(self, **params):
self.myparams = params['myparams']
params.pop('myparams')
 self.myinit(**params)
 return self
def get_params(self, deep=True):
params = {'myparams': self.myparams}
return params
def apply(self, X):
return self.clf.apply(X)
 def decision_path(self, X):
return self.clf.decision_path(X)
 def staged_decision_function(self, X):
return self.clf.staged_decision_function(X)
 def staged_predict(self, X):
return self.clf.staged_predict(X)
 def staged_predict_proba(self, X):
return self.clf.staged_predict_proba(X)
 def staged_score(self, X):
return self.clf.staged_score(X )<define_variables>
df_train = pd.read_csv(".. /input/train.csv", encoding = 'ISO-8859-1')
 df_subm = pd.read_csv(".. /input/test.csv", encoding = 'ISO-8859-1' )
def seed_all() :
random.seed(SEED)
 np.random.seed(SEED)
 random.seed(SEED)
 seed_all()<load_from_csv>
df_train.isnull().sum().sum() , df_subm.isnull().sum().sum()
def load_preprocess_data(filename='.. /input/jane-street-market-prediction/train.csv', isTrainData=True):
dtype = None
if isTrainData:
dtype = {
'date' : 'int64',
'weight' : 'float64',
'resp' : 'float64',
'ts_id' : 'int64',
'feature_0' : 'float64'
}
else:
dtype = {
'date' : 'int64',
'weight' : 'float64',
'feature_0' : 'float64'
}
for i in range(1, 130):
k = 'feature_' + str(i)
 dtype[k] = 'float32'
X = pd.read_csv(filename, dtype=dtype)
 resp_cols = ['resp_1', 'resp_2', 'resp_3','resp_4', 'resp']
X = X.query('date > 85')
 X = X[X['weight'] != 0].reset_index(drop = True)
 y = np.stack([(X[c] > 0 ).astype('int')for c in resp_cols] ).T
f_columns = [c for c in X.columns if "feature" in c]
Weights = X['weight'].values.reshape(( -1,1))
if isTrainData:
X.drop(columns=['date', 'weight', 'resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'ts_id'], inplace=True)
 else:
X.drop(columns=['date', 'weight'], inplace=True)
 preprocess_pipe = Pipeline([ 
("imputer", SimpleImputer(missing_values=np.nan, strategy='mean')) ,
])
 X = preprocess_pipe.fit_transform(X)
 X = np.hstack(( X, Weights))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05)
 del X, y
gc.collect()
W_train = X_train[:, -1]
X_train = X_train[:, :-1]
W_test = X_test[:, -1]
X_test = X_test[:, :-1]
return X_train, X_test, y_train, y_test, W_train, W_test, preprocess_pipe<train_model>
X_train = df_train[df_train.columns[1:]]
y_train = df_train['label']
X_TRAIN, X_TEST, Y_TRAIN, Y_TEST, W_train, W_test, preprocess_pipe = load_preprocess_data()
gc.collect()
X_TRAIN.shape, Y_TRAIN.shape<normalization>
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train )
def learning_rate_010_decay_power_09(current_iter):
base_learning_rate = 0.1
lr = base_learning_rate * np.power (.995, current_iter)
 return lr if lr > 1e-2 else 1e-2 
<init_hyperparams>
y_train = to_categorical(y_train)
 y_test = to_categorical(y_test )
FIT_PARAMS= {
"early_stopping_rounds":30,
"eval_metric" : 'auc',
"eval_set" : [(X_TEST, Y_TEST[:,-1])],
'eval_names': ['valid'],
'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_010_decay_power_09)],
'verbose': 50,
'categorical_feature': 'auto'
}<init_hyperparams>
X_train = X_train/255
X_test = X_test/255
<init_hyperparams>
model = Sequential()
OPT_PARAMS_1 = {'n_estimators': N_ESTIMATORS, 'colsample_bytree': 0.668, 'min_child_samples': 150, 'min_child_weight': 1, 'num_leaves': 80, 'reg_alpha': 0, 'reg_lambda': 0.002, 'subsample': 0.87}
OPT_PARAMS_2 = {'n_estimators': N_ESTIMATORS, 'colsample_bytree': 0.668, 'min_child_samples': 190, 'min_child_weight': 1, 'num_leaves': 90, 'reg_alpha': 0, 'reg_lambda': 0.002, 'subsample': 0.87}<train_model>
model.add(Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1)))
 model.add(Conv2D(32,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
def create_train_lgbm(X_train, y_train, component):
if component == 1:
opt_params = deepcopy(OPT_PARAMS_1)
 else:
opt_params = deepcopy(OPT_PARAMS_2)
 lgb_clf_1 = lgb.LGBMClassifier(**opt_params)
 lgb_clf_1.fit(X_train, y_train, **FIT_PARAMS)
 if lgb_clf_1.best_iteration_ != N_ESTIMATORS:
opt_params['n_estimators'] = lgb_clf_1.best_iteration_
lgb_clf_1 = lgb.LGBMClassifier(**opt_params)
 lgb_clf_1.fit(X_train, y_train, **FIT_PARAMS)
 X_temp = X_train[10].reshape(( 1, -1))
np.round(lgb_clf_1.predict(X_temp)).astype(int)
 return lgb_clf_1<train_model>
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
def getLgbs() :
LGBS = []
for model_id in range(5):
y_train = Y_TRAIN[:, model_id]
lgbm_1 = create_train_lgbm(X_TRAIN, y_train, 1)
 lgbm_2 = create_train_lgbm(X_TRAIN, y_train, 2)
 LGBS.append(( lgbm_1, lgbm_2))
pickleSave(LGBS, 'lgbs.bin')
 return LGBS<choose_model_class>
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True )
def getSclfs() :
SCLFS = []
for model_id in range(5):
sclf = StackingClassifier(classifiers=LGBS[model_id], fit_base_estimators=False,
use_probas=True, average_probas=False,
meta_classifier=Best_clf_cv_transformer({ 'name': 'LSvc', 'params': {'penalty': 'l2', 'class_weight': 'balanced'}, 'param_grid': {'C' : [0.01, 0.05, 0.1, 1]} }))
sclf.name = 'sclf_' + str(model_id)
 y_train = Y_TRAIN[:, model_id]
sclf.fit(X_TRAIN, y_train)
 SCLFS.append(sclf)
 pickleSave(SCLFS, 'sclfs.bin')
 X_temp = X_TRAIN[10].reshape(( 1, -1))
np.round(SCLFS[0].predict(X_temp)).astype(int)
 return SCLFS<load_pretrained>
model.compile(loss='categorical_crossentropy', optimizer=sgd,metrics=['accuracy'] )
LGBS = unpickle('.. /input/jane-lgbm-stackedlsvc/lgbs.bin')
 SCLFS = unpickle('.. /input/jane-lgbm-stackedlsvc/sclfs.bin' )<predict_on_test>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
)
def predict(test_df, isRetProb=False):
test_df.drop(columns=['weight', 'date'], inplace=True)
 test_df.reset_index(drop=True, inplace=True)
 X_test = preprocess_pipe.transform(test_df ).reshape(( -1, 130))
y_probs = []
for sclf in SCLFS:
y_p = sclf.predict_proba(X_test ).reshape(( -1, 2)) [:, 1].reshape(( -1, 1))
y_probs.append(y_p)
 y_probs = np.hstack(y_probs)
 pred_pr = np.median(y_probs, axis=1)
 y_pred =(pred_pr >= 0.5 ).astype(int)
 if isRetProb:
return y_pred, pred_pr
else:
return y_pred<define_variables>
datagen.fit(X_train )
env = janestreet.make_env()
env_iter = env.iter_test()<predict_on_test>
model.fit_generator(datagen.flow(X_train, y_train, batch_size=128),
steps_per_epoch=int(len(X_train)/ 128), epochs=30 )
for test_df, pred_df in env_iter:
if test_df["weight"].item() > 0:
predictions = predict(test_df)
 pred_df.action = predictions
else:
pred_df.action = 0
env.predict(pred_df )<train_model>
test_data = df_subm.values
print('Done !' )<load_from_csv>
test_data = test_data.reshape(test_data.shape[0],28,28,1 )
if tuning or training:
train_data = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train_data.fillna(train_data.mean() ,inplace=True)
 metadata = pd.read_csv('/kaggle/input/jane-street-market-prediction/features.csv')
 metadata.drop(['feature'],axis=1,inplace=True)
 def replace_bool(tf):
if tf:
return 1
else:
return 0
metadata = metadata.applymap(replace_bool)
 metadata_norm = metadata/metadata.sum()
metadata_norm = metadata_norm.applymap(np.sqrt)
 features_transform = metadata_norm.values
start_date=86
feature_columns = [col for col in train_data.columns.values if 'feature' in col]
X_m = np.matmul(train_data[train_data['date']>=start_date][feature_columns].values,features_transform)
 corr=abs(train_data[feature_columns].corr())
 ordered_feature_columns=[feature_columns.pop(0)]
for col in feature_columns:
corr_max = corr[col][ordered_feature_columns].idxmax()
corr_max_idx = ordered_feature_columns.index(corr_max)
 ordered_feature_columns.insert(corr_max_idx+1,col)
 f_mean = train_data[ordered_feature_columns].mean()
f_mean.to_csv('f_mean.csv',index=False)
 with open("ordered_columns.txt", 'wb')as f:
pickle.dump(( ordered_feature_columns), f)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
train_data = train_data[train_data['date']>=start_date]
X = np.concatenate(( train_data[ordered_feature_columns].values,X_m),axis=1)
 y = np.stack([(train_data[c] > 0 ).astype('int')for c in resp_cols] ).T
max_date = train_data['date'].max()
split_date = int(0.7*(max_date-start_date))
train_test_split = train_data['date'][train_data['date']==split_date].index[0]
del train_data
X_train=X[:train_test_split,:]
X_test=X[train_test_split:,:]
y_train=y[:train_test_split,:]
y_test=y[train_test_split:,:] 
<choose_model_class>
test_data = test_data/255
tf.random.set_seed(42)
 SEED=42
def create_model(hp, num_columns, num_labels):
inp = tf.keras.layers.Input(shape =(num_columns, 1))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Conv1D(filters=8,
kernel_size=hp.Int('kernel_size',5,10,step=5),
strides=1,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 x = tf.keras.layers.Flatten()(x)
 for i in range(hp.Int('num_layers',12,16)) :
x = tf.keras.layers.Dense(hp.Int(f'num_units_{i}',32,64))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(hp.Float(f'dropout_{i}',0,0.5))(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = hp.Float('lr',0.00001,0.1,default=0.001)) ,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.01),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<train_model>
predictions = model.predict(test_data )
if tuning:
model_fn = lambda hp: create_model(hp,X_train.shape[-1],y_train.shape[-1])
 tuner = kt.tuners.bayesian.BayesianOptimization(
 hypermodel=model_fn,
objective= kt.Objective('val_AUC', direction='max'),
num_initial_points=4,
max_trials=20)
 tuner.search(X_train,y_train,batch_size=4096,epochs=20, validation_data =(X_test, y_test), callbacks=[EarlyStopping('val_AUC', mode='max',patience=5)])
 hp = tuner.get_best_hyperparameters(1)[0]
pd.to_pickle(hp,'best_hp_cnn_day_86_metadata_deep.pkl' )<train_model>
predictions = np.argmax(predictions, axis=1, out=None )
if training:
hp = pd.read_pickle('best_hp_cnn_day_86_metadata_deep.pkl')
 model_fn = lambda hp: create_model(hp,X_train.shape[-1],y_train.shape[-1])
 model = model_fn(hp)
 model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=100,batch_size=4096,
callbacks=[EarlyStopping('val_AUC',mode='max',patience=10,restore_best_weights=True)])
 model.save_weights('JS_CNN_day_86_metadata_deep.hdf5' )<load_from_csv>
with open("resultCNNwithPrepros.csv", "wb")as f:
f.write(b'ImageId,Label
')
 np.savetxt(f, np.hstack([(np.array(range(28000)) +1 ).reshape(-1,1), predictions.astype(int ).reshape(-1,1)]), fmt='%i', delimiter="," )