Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<import_modules>
plt.style.use('ggplot')
 %matplotlib inline
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm, tqdm_notebook
from sklearn.covariance import EmpiricalCovariance
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import sympy
from sklearn import svm, neighbors, linear_model, neural_network
from xgboost import XGBClassifier
from sklearn.covariance import *
from sklearn.utils.validation import check_random_state
from sklearn.mixture import *
from sklearn.cluster import *<load_from_csv>
seed = 33
np.random.RandomState(seed)
 valid_size = 0.1
to_augment = False
%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train.head()<set_options>
data_path = '.. /input/Kannada-MNIST/'
train_path = data_path + 'train.csv'
test_path = data_path + 'test.csv'
dig_path = data_path + 'Dig-MNIST.csv'
sample_path = data_path + 'sample_submission.csv'
save_path = ''
load_path = '.. /input/kennada-mnist-pretrained-model/'
warnings.filterwarnings('ignore' )<categorify>
train_df = pd.read_csv(train_path)
 test_df = pd.read_csv(test_path)
 dig_df = pd.read_csv(dig_path)
 sample_df = pd.read_csv(sample_path )
def dist(array, centre):
x=float(0)
 for i in range(len(array)) :
x +=(( array[i]-centre[i])*(array[i]-centre[i]))
return x
def get_c(data):
data.drop('labels', axis=1, inplace=True)
 centre = [1]*data.shape[1]
for i in range(data.shape[1]):
if data[i].mean() < 0:
centre[i] = -1
return(centre)
 def my_min(a, b, c):
if a<b:
if a<c: return a
return c
if b<c: return b
return c 
<predict_on_test>
X = train_df.drop('label', axis=1 ).to_numpy()
y = train_df['label'].to_numpy()
X_dig = dig_df.drop('label', axis=1 ).to_numpy()
y_dig = dig_df['label'].to_numpy()
X_test = test_df.drop('id', axis=1 ).to_numpy()
X = X.reshape(-1, 28, 28, 1)
 y = to_categorical(y)
 X_dig = X_dig.reshape(-1, 28, 28, 1)
 X_test = X_test.reshape(-1, 28, 28, 1)
 X = X / 255
X_dig = X_dig / 255
X_test = X_test / 255
print('X shape is {}'.format(X.shape))
print('y shape is {}'.format(y.shape))
print('X_dig shape is {}'.format(X_dig.shape))
print('y_dig shape is {}'.format(y_dig.shape))
print('X_test shape is {}'.format(X_test.shape))
def classify(data, val, test, y):
data = pd.DataFrame(train3)
 data['target'] = y
zero = data[data['target'] == 1]
one = data[data['target'] == 0]
zero.drop('target', axis=1, inplace=True)
 one.drop('target', axis=1, inplace=True)
 clf = KMeans(n_clusters=3)
 labels = clf.fit_predict(zero)
 zero['labels'] = labels
zero_0 =(zero[zero['labels'] == 0])
 zero_1 =(zero[zero['labels'] == 1])
 zero_2 =(zero[zero['labels'] == 2])
 clf = KMeans(n_clusters=3)
 labels = clf.fit_predict(one)
 one['labels'] = labels
one_0 =(one[one['labels'] == 0])
 one_1 =(one[one['labels'] == 1])
 one_2 =(one[one['labels'] == 2])
 c_z_0 = get_c(zero_0)
 c_z_1 = get_c(zero_1)
 c_z_2 = get_c(zero_2)
 c_o_0 = get_c(one_0)
 c_o_1 = get_c(one_1)
 c_o_2 = get_c(one_2)
 pred_val = [0]*val.shape[0]
for i in range(val.shape[0]):
array = val.loc[i]
dist0_0 = dist(array, c_z_0)
 dist0_1 = dist(array, c_z_1)
 dist0_2 = dist(array, c_z_2)
 dist1_0 = dist(array, c_o_0)
 dist1_1 = dist(array, c_o_1)
 dist1_2 = dist(array, c_o_2)
 aggr =(dist0_0+dist0_1+dist0_2 +dist1_0+dist1_1+dist1_2)/3
dist1 =(dist1_0+dist1_1+dist1_2)/ 3
dist0 =(dist0_0+dist0_1+dist0_2)/ 3
pred_val[i] = 1-1/np.exp(dist1/aggr)
 pred_test = [0]*test.shape[0]
for i in range(test.shape[0]):
array = test.loc[i]
dist0_0 = dist(array, c_z_0)
 dist0_1 = dist(array, c_z_1)
 dist0_2 = dist(array, c_z_2)
 dist1_0 = dist(array, c_o_0)
 dist1_1 = dist(array, c_o_1)
 dist1_2 = dist(array, c_o_2)
 aggr =(dist0_0+dist0_1+dist0_2 +dist1_0+dist1_1+dist1_2)/4
dist1 =(dist1_0+dist1_1+dist1_2)/ 3
dist0 =(dist0_0+dist0_1+dist0_2)/ 3
pred_test[i] = 1-1/np.exp(dist1/aggr)
 return np.array(pred_val), np.array(pred_test)
<prepare_x_and_y>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=valid_size, random_state=seed)
 print('X_train shape = {}'.format(X_train.shape))
print('Y_train shape = {}'.format(y_train.shape))
print('X_valid shape = {}'.format(X_valid.shape))
print('Y_valid shape = {}'.format(y_valid.shape))
oof = np.zeros(len(train))
preds = np.zeros(len(test))
cols = [c for c in train.columns if c not in ['id', 'target']]
for k in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
test2 = test[test['wheezy-copper-turtle-magic']==k]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for i,(train_index, test_index)in enumerate(skf.split(train3, train2['target'])) :
oof[idx1[test_index]], test_pred = classify(pd.DataFrame(train3[train_index,:]), pd.DataFrame(train3[test_index,:]), pd.DataFrame(test3), train2.loc[train_index]['target'])
 preds[idx2] += test_pred / skf.n_splits
print(roc_auc_score(train2.loc[test_index]['target'], oof[idx1[test_index]]))
if k==5: break
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
<import_modules>
def build_model(optimizer):
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
 return model
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
def save_trained_model(model, save_path, optimizer):
model_json = model.to_json()
with open('{}Kennada MNIST with {}.json'.format(save_path, optimizer), "w")as json_file:
json_file.write(model_json)
 model.save_weights('{}Kennada MNIST with {}.h5'.format(save_path, optimizer))
def load_trained_model(optimizers, optimizer, load_path):
json_file = open('{}Kennada MNIST with {}.json'.format(load_path, optimizers[optimizer]), 'r')
 loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
 model.load_weights('{}Kennada MNIST with {}.h5'.format(load_path, optimizers[optimizer]))
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
 return model
RANDOM_SEED = 4123
cols = [
c for c in train.columns
if c not in ['id', 'target', 'wheezy-copper-turtle-magic']
]
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps<save_to_csv>
def load_history(load_path, optimizer):
history = pd.read_csv('{}Kennada MNIST with {}.csv'.format(load_path, optimizer))
return history.to_dict('list')
 def save_history(history, save_path, optimizer):
hist_df = pd.DataFrame(history)
 hist_df.to_csv('{}Kennada MNIST with {}.csv'.format(save_path, optimizer), index=False )
%%time
SKIP_COMMIT = True
if SKIP_COMMIT:
sub = pd.read_csv('.. /input/sample_submission.csv')
 if sub.shape[0] < 200000:
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.to_csv('submission.csv', index=False)
 raise ValueError('Stop!!!')
 oof_nusvc = np.zeros(len(train))
preds_nusvc = np.zeros(len(test))
oof_nb= np.zeros(len(train))
preds_nb = np.zeros(len(test))
oof_lr = np.zeros(len(train))
preds_lr = np.zeros(len(test))
oof_qda = np.zeros(len(train))
preds_qda = np.zeros(len(test))
oof_lp = np.zeros(len(train))
preds_lp = np.zeros(len(test))
oof_lgbm = np.zeros(len(train))
preds_lgbm = np.zeros(len(test))
oof_gm = np.zeros(len(train))
preds_gm = np.zeros(len(test))
oof_rf = np.zeros(len(train))
preds_rf = np.zeros(len(test))
params_lgbm_1 = {
'boosting_type': 'gbdt',
'objective': 'xentropy',
'metric': ['auc'],
'num_leaves': 31,
'learning_rate': 0.5,
'feature_fraction': 1.0,
'bagging_fraction': 1.0,
'bagging_freq': 18,
'num_threads': 8,
'lambda_l2': 5.0,
'max_bin': 3
}
for i in range(512):
print(i, end=' ')
 train2 = train[train['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx2 = test2.index
data = pd.concat(
 [
train2,
test2
],
axis=0 
)
 train2.reset_index(drop=True, inplace=True)
 train_size = train2.shape[0]
sel = VarianceThreshold(threshold=1.5)
 tmp = sel.fit_transform(
 data[cols] 
)
 train3 = tmp[:train_size, :]
test3 = tmp[train_size:, :]
ss = StandardScaler()
tmp_scaled = ss.fit_transform(tmp)
 train3_scaled = tmp_scaled[:train_size, :]
test3_scaled = tmp_scaled[train_size:, :]
poly = PolynomialFeatures(degree=2)
 tmp_poly = poly.fit_transform(tmp_scaled)
 train3_poly = tmp_poly[:train_size, :]
test3_poly = tmp_poly[train_size:, :]
gm_clf_4 = mixture.GaussianMixture(
 n_components=4,
random_state=RANDOM_SEED 
)
 gm_tmp_4 = gm_clf_4.fit_predict(tmp ).reshape(-1, 1)
 le_4 = OneHotEncoder()
gm_tmp_4 = le_4.fit_transform(gm_tmp_4 ).todense()
gm_train3_4 = gm_tmp_4[:train_size, :]
gm_test3_4 = gm_tmp_4[train_size:, :]
gm_clf_6 = mixture.GaussianMixture(
 n_components=6,
random_state=RANDOM_SEED 
)
 gm_tmp_6 = gm_clf_6.fit_predict(tmp ).reshape(-1, 1)
 le_6 = OneHotEncoder()
gm_tmp_6 = le_6.fit_transform(gm_tmp_6 ).todense()
gm_train3_6 = gm_tmp_6[:train_size, :]
gm_test3_6 = gm_tmp_6[train_size:, :]
skf = StratifiedKFold(
 n_splits=11,
random_state=RANDOM_SEED,
shuffle=True 
)
 for train_index, test_index in skf.split(train3, train2['target']):
train_train_index, train_val_index = train_test_split(
 train_index,
test_size=0.3,
random_state=RANDOM_SEED 
)
 train_dataset = lgb.Dataset(
 np.hstack(
(
 train3_scaled[train_train_index,:],
gm_tmp_4[:train_size, :][train_train_index, :].tolist() ,
gm_tmp_6[:train_size, :][train_train_index, :].tolist() ,
train3_poly[train_train_index,:] 
)
),
train2.loc[train_train_index]['target'],
free_raw_data=False 
)
 valid_dataset = lgb.Dataset(
 np.hstack(
(
 train3_scaled[train_val_index,:],
gm_tmp_4[:train_size, :][train_val_index, :].tolist() ,
gm_tmp_6[:train_size, :][train_val_index, :].tolist() ,
train3_poly[train_val_index,:] 
)
),
train2.loc[train_val_index]['target'],
free_raw_data=False 
)
 gm = lgb.train(
 params_lgbm_1,
train_dataset,
num_boost_round=1000,
early_stopping_rounds=20,
valid_sets=(train_dataset, valid_dataset),
valid_names=('train', 'valid'),
feature_name=[str(l)for l in range(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_tmp_4[:train_size, :][test_index, :].tolist() ,
gm_tmp_6[:train_size, :][test_index, :].tolist() ,
train3_poly[test_index,:] 
)
 ).shape[1] 
)],
categorical_feature=[str(l)for l in
range(
 train3_scaled.shape[1],
train3_scaled.shape[1] + gm_tmp_4.shape[1] + gm_tmp_6.shape[1] 
)
 ],
verbose_eval=0 
)
 oof_lgbm[idx1[test_index]] = gm.predict(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_tmp_4[:train_size, :][test_index, :].tolist() ,
gm_tmp_6[:train_size, :][test_index, :].tolist() ,
train3_poly[test_index,:] 
)
)
)
 preds_lgbm[idx2] += gm.predict(
 np.hstack(
(
 test3_scaled,
gm_test3_4.tolist() ,
gm_test3_6.tolist() ,
test3_poly 
)
)
)/ skf.n_splits
ms, ps = get_mean_cov(
 train3[train_index, :],
train2.loc[train_index]['target'].values 
)
 gm = mixture.GaussianMixture(
 n_components=2,
init_params='random',
covariance_type='full',
tol=0.001,
reg_covar=0.001,
max_iter=100,
n_init=1,
means_init=ms,
precisions_init=ps,
random_state=RANDOM_SEED 
)
 gm.fit(tmp)
 oof_gm[idx1[test_index]] = gm.predict_proba(
 train3[test_index,:] 
)[:, 0]
preds_gm[idx2] += gm.predict_proba(
 test3 
)[:, 0] / skf.n_splits
lp = LabelPropagation(
 kernel='rbf',
gamma=0.15301581563198507,
n_jobs=-1 
)
 lp.fit(
 train3_scaled[train_index,:],
train2.loc[train_index]['target'] 
)
 oof_lp[idx1[test_index]] = lp.predict_proba(
 train3_scaled[test_index, :] 
)[:,1]
preds_lp[idx2] += lp.predict_proba(
 test3_scaled 
)[:,1] / skf.n_splits
clf = NuSVC(
 probability=True,
kernel='poly',
degree=2,
gamma='auto',
random_state=RANDOM_SEED,
nu=0.27312143533915767,
coef0=0.4690615598786931 
)
 clf.fit(
 np.hstack(
(
 train3_scaled[train_index,:],
gm_train3_4[train_index, :],
gm_train3_6[train_index, :] 
)
),
train2.loc[train_index]['target'] 
)
 oof_nusvc[idx1[test_index]] = clf.predict_proba(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_train3_4[test_index, :],
gm_train3_6[test_index, :] 
)
)
)[:,1]
preds_nusvc[idx2] += clf.predict_proba(
 np.hstack(
(
 test3_scaled,
gm_test3_4,
gm_test3_6 
)
)
)[:,1] / skf.n_splits
clf = RandomForestClassifier(
 max_depth=4,
n_jobs=-1,
n_estimators=20 
)
 clf.fit(
 np.hstack(
(
 train3_scaled[train_index,:],
gm_train3_4[train_index, :],
gm_train3_6[train_index, :] 
)
),
train2.loc[train_index]['target'] 
)
 oof_rf[idx1[test_index]] = clf.predict_proba(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_train3_4[test_index, :],
gm_train3_6[test_index, :] 
)
)
)[:,1]
preds_rf[idx2] += clf.predict_proba(
 np.hstack(
(
 test3_scaled,
gm_test3_4,
gm_test3_6 
)
)
)[:,1] / skf.n_splits
clf = QuadraticDiscriminantAnalysis(
 reg_param=0.5674164995882528 
)
 clf.fit(
 train3[train_index,:],
train2.loc[train_index]['target'] 
)
 oof_qda[idx1[test_index]] += clf.predict_proba(
 train3[test_index, :] 
)[:,1]
preds_qda[idx2] += clf.predict_proba(
 test3 
)[:,1] / skf.n_splits
clf = linear_model.LogisticRegression(
 solver='saga',
penalty='l2',
C=0.01,
tol=0.001,
random_state=RANDOM_SEED 
)
 clf.fit(
 train3_poly[train_index,:],
train2.loc[train_index]['target'] 
)
 oof_lr[idx1[test_index]] = clf.predict_proba(
 train3_poly[test_index,:] 
)[:,1]
preds_lr[idx2] += clf.predict_proba(
 test3_poly 
)[:,1] / skf.n_splits
clf = GaussianNB()
clf.fit(
 np.hstack(
(
 train3_scaled[train_index,:],
gm_train3_6[train_index, :],
gm_train3_4[train_index, :] 
)
),
train2.loc[train_index]['target'] 
)
 oof_nb[idx1[test_index]] = clf.predict_proba(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_train3_6[test_index, :],
gm_train3_4[test_index, :] 
)
)
)[:,1]
preds_nb[idx2] += clf.predict_proba(
 np.hstack(
(
 test3_scaled,
gm_test3_6,
gm_test3_4 
)
)
)[:,1] / skf.n_splits
print('
svcnu', roc_auc_score(train['target'], oof_nusvc))
print('gm', roc_auc_score(train['target'], oof_gm))
print('qda', roc_auc_score(train['target'], oof_qda))
print('log reg poly', roc_auc_score(train['target'], oof_lr))
print('gnb', roc_auc_score(train['target'], oof_nb))
print('lp', roc_auc_score(train['target'], oof_lp))
print('lgbm', roc_auc_score(train['target'], oof_lgbm))
print('rf', roc_auc_score(train['target'], oof_rf))
oof_qda = oof_qda.reshape(-1, 1)
 preds_qda = preds_qda.reshape(-1, 1)
 oof_lr = oof_lr.reshape(-1, 1)
 preds_lr = preds_lr.reshape(-1, 1)
 oof_nusvc = oof_nusvc.reshape(-1, 1)
 preds_nusvc = preds_nusvc.reshape(-1, 1)
 oof_nb = oof_nb.reshape(-1, 1)
 preds_nb = preds_nb.reshape(-1, 1)
 oof_lp = oof_lp.reshape(-1, 1)
 preds_lp = preds_lp.reshape(-1, 1)
 oof_gm = oof_gm.reshape(-1, 1)
 preds_gm = preds_gm.reshape(-1, 1)
 oof_lgbm = oof_lgbm.reshape(-1, 1)
 preds_lgbm = preds_lgbm.reshape(-1, 1)
 oof_rf = oof_rf.reshape(-1, 1)
 preds_rf = preds_rf.reshape(-1, 1)
 tr_2 = np.concatenate(
(
 oof_qda,
oof_nusvc,
oof_lr,
oof_nb,
oof_lp,
oof_gm,
oof_lgbm,
oof_rf 
),
axis=1 
)
 te_2 = np.concatenate(
(
 preds_qda,
preds_nusvc,
preds_lr,
preds_nb,
preds_lp,
preds_gm,
preds_lgbm,
preds_rf 
),
axis=1 
)
 print(np.corrcoef(tr_2, rowvar=False))
params = {
'boosting_type': 'gbdt',
'objective': 'xentropy',
'metric': ['auc'],
'num_leaves': 3,
'learning_rate': 0.1,
'feature_fraction': 0.4,
'bagging_fraction': 0.4,
'bagging_freq': 5,
'num_threads': 8
}
params.update(
 {
'bagging_fraction': 0.9687497922020039,
'bagging_freq': 100,
'feature_fraction': 0.7578027095458152,
'lambda_l2': 4.871836452096843,
'learning_rate': 0.41230192513715164,
'max_bin': 20,
'num_leaves': 8
} 
)
 oof_boosting_2_bad_cv = np.zeros(train.shape[0])
 pred_te_boosting_2_bad_cv = np.zeros(test.shape[0])
 train2 = train.copy()
train2.reset_index(drop=True,inplace=True)
 skf = StratifiedKFold(
 n_splits=11,
random_state=RANDOM_SEED,
shuffle=True 
)
 for train_index, test_index in skf.split(tr_2, train2['target']):
train_dataset = lgb.Dataset(
 np.hstack(
(
 tr_2[train_index, :],
tr_2[train_index, :] ** 2.587508645172711 
)
),
train2['target'][train_index],
free_raw_data=False 
)
 valid_dataset = lgb.Dataset(
 np.hstack(
(
 tr_2[test_index, :] ,
tr_2[test_index, :] ** 2.587508645172711 
)
),
train2['target'][test_index],
free_raw_data=False 
)
 gbm = lgb.train(
 params,
train_dataset,
num_boost_round=1000,
early_stopping_rounds=100,
valid_sets=(train_dataset, valid_dataset),
valid_names=('train', 'valid'),
verbose_eval=100 
)
 oof_boosting_2_bad_cv[test_index] = gbm.predict(
 np.hstack(
(
 tr_2[test_index, :],
tr_2[test_index, :] ** 2.587508645172711 
)
)
)
 pred_te_boosting_2_bad_cv += gbm.predict(
 np.hstack(
(
 te_2,
te_2 ** 2.587508645172711 
)
)
)/ skf.n_splits
print('gnb', roc_auc_score(train['target'], oof_boosting_2_bad_cv))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = pred_te_boosting_2_bad_cv
sub.to_csv('submission.csv', index=False )<data_type_conversions>
batch_size = 1024
verbose = 0
epochs = 30
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps 
<sort_values>
optimizers = {
'sgd': 'SGD',
'rmsprop': 'RMSprop',
'adagrad': 'Adagrad',
'adadelta': 'Adadelta',
'adam': 'Adam',
'adamax': 'Adamax',
'nadam': 'Nadam',
}
learning_rates = {
'sgd': 1e-2,
'rmsprop': 1e-3,
'adagrad': 1e-2,
'adadelta': 1.0,
'adam': 1e-3,
'adamax': 2e-3,
'nadam': 2e-3,
}
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 ms, ps = get_mean_cov(train3,train2['target'].values)
 skf = StratifiedKFold(n_splits=15, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
P = train3[train_index,:]
T = train2.loc[train_index]['target'].values
gm = GaussianMixture(n_components=2, init_params='kmeans', covariance_type='full', tol=0.1,reg_covar=0.1, max_iter=150, n_init=5,means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([P,test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:,0]
preds[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5)) 
<save_to_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=0,
factor=0.5,
min_lr=0.00001)
 train_datagen = ImageDataGenerator(rescale=1.0,
rotation_range=10,
width_shift_range=0.25,
height_shift_range=0.25,
shear_range=0.1,
zoom_range=0.25,
horizontal_flip=False)
 valid_datagen = ImageDataGenerator(rescale=1.0 )
sub = pd.read_csv('.. /input/instant-gratification/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<load_from_csv>
history = {}
model = {}
for n, optimizer in enumerate(optimizers):
model[optimizer] = build_model(optimizer)
 start = time.time()
if to_augment:
h = model[optimizer].fit_generator(train_datagen.flow(X_train, y_train, batch_size=batch_size),
steps_per_epoch=100,
epochs=epochs,
validation_data=valid_datagen.flow(X_valid, y_valid),
callbacks=[learning_rate_reduction],
verbose=verbose)
 else:
h = model[optimizer].fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_valid,y_valid),
callbacks=[learning_rate_reduction],
verbose=verbose)
 history[optimizer] = h.history
print("{0} Optimizer: ".format(optimizers[optimizer]))
print("Epochs={0:d}, Train accuracy={1:.5f}, Validation accuracy={2:.5f}, Training time={3:.2f} minutes" 
.format(epochs,
max(history[optimizer]['accuracy']),
max(history[optimizer]['val_accuracy']), 
(time.time() -start)/60))
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}')
 sub1 = pd.read_csv('.. /input/sample_submission.csv')
 sub1['target'] = preds<import_modules>
def get_best_optimizer(history, optimizers, to_smooth=False, filter_n=3):
best_val_scores = np.zeros(( len(optimizers),))
for n, h in enumerate(history.values()):
val_acc = h['val_accuracy']
if to_smooth:
val_acc = smoothing_filter(val_acc, filter_n)
 best_val_scores[n] = np.max(val_acc)
 return list(optimizers.keys())[np.argmax(best_val_scores)]
FOLD = 0
SEED = 1337
NOTIFY_EACH_EPOCH = False
WORKERS = 0
BATCH_SIZE = 512
N_SPLITS = 10
np.random.seed(SEED)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def save_checkpoint(state):

 print(" Saving checkpoint")
 filename = f'./checkpoint-{state["epoch"]}.pt.tar'
torch.save(state, filename)
 def initialize(model, path=None, optimizer=None):
if path == None:
checkpoints = glob.glob('./*.pt.tar')
 path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0])for checkpoint in checkpoints])]
checkpoint = torch.load(path)
 model.load_state_dict(checkpoint['model'])
 print(f' Loaded checkpoint {path} | Trained for {checkpoint["epoch"] + 1} epochs')
 if optimizer:
optimizer.load_state_dict(checkpoint['optimizer'])
 epoch = checkpoint['epoch'] + 1
train_iteration = checkpoint['train_iteration']
val_iteration = checkpoint['val_iteration']
return model, optimizer, epoch, train_iteration, val_iteration
else:
return model<load_from_csv>
best_optimizer = get_best_optimizer(history, optimizers, to_smooth=True, filter_n=7)
 print("Optimizer with best validation score is '{}'.".format(optimizers[best_optimizer]))
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<save_to_csv>
additional_epochs = 50
kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 train_idx, val_idx = list(kfold.split(train)) [FOLD]
x_train, x_val = train.iloc[train_idx], train.iloc[val_idx]
x_train.to_csv('train.csv')
 x_val.to_csv('val.csv' )<string_transform>
filepath = save_path + 'best_model_with_'+ best_optimizer + '_on_' + str(additional_epochs)+ '.hdf5'
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=0, save_best_only=True, mode='max')
 new_model = build_model(best_optimizer)
 start = time.time()
new_history = new_model.fit_generator(train_datagen.flow(X_train, y_train, batch_size=batch_size),
steps_per_epoch=100,
epochs=additional_epochs,
validation_data=valid_datagen.flow(X_valid, y_valid),
callbacks=[learning_rate_reduction, checkpoint],
verbose=verbose)
 new_history = new_history.history
print("{0} Optimizer: ".format(optimizers[best_optimizer]))
print("Epochs={0:d}, Train accuracy={1:.5f}, Validation accuracy={2:.5f}, Training time={3:.2f} minutes" 
.format(additional_epochs,
max(new_history['accuracy']),
max(new_history['val_accuracy']), 
(time.time() -start)/60))
nlp = English()
def tokenize(sentence):
x = nlp(sentence)
 return [token.text for token in x]<load_from_csv>
def make_prediction(model, x):
y_pred = model.predict(x)
 return np.argmax(y_pred, axis=1 )
<categorify><EOS> 
y_result = make_prediction(new_model, X_test)
 sample_df['label'] = y_result
sample_df.to_csv('submission.csv',index=False )