Kaggle competition: Instant Gratification
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST-CNN-ENSEMBLE.csv",index=False )<import_modules>
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )
from keras import models
from keras import layers
from keras.layers import Dense, Conv2D, Flatten,MaxPooling2D,Dropout,ZeroPadding2D,BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
import pandas as pd
import numpy as np
from matplotlib import pyplot
import os
import zipfile
import tensorflow as tf
from tensorflow.keras.optimizers import RMSprop,Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import random 
<load_from_csv>
null_cnt = train.isnull().sum().sort_values()
print('null count:', null_cnt[null_cnt > 0] )
Train = pd.read_csv(".. /input/train.csv")
 Test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
print(train['wheezy-copper-turtle-magic'].describe())
 print()
print('unique value count:', train['wheezy-copper-turtle-magic'].nunique() )
y_train = Train['label'].values
train_images = Train.loc[:,Train.columns != 'label'].values
train_images = train_images.reshape(-1,28,28,1)
 train_images.shape 
<data_type_conversions>
numcols = train.drop(['id','target','wheezy-copper-turtle-magic'],axis=1 ).select_dtypes(include='number' ).columns.values
train_images = train_images.astype('float32')/255
test_images_final = test_images_final.astype('float32')/255<choose_model_class>
pca = PCA()
pca.fit(train[numcols])
 plt.xlabel('components')
 plt.plot(np.add.accumulate(pca.explained_variance_ratio_))
plt.show()
network = tf.keras.models.Sequential([
tf.keras.layers.ZeroPadding2D(padding=(1, 1), data_format=None,input_shape=(28, 28, 1)) ,
tf.keras.layers.Conv2D(32,(5,5), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ZeroPadding2D(padding=(1, 1), data_format=None),
tf.keras.layers.Conv2D(64,(6,6), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ZeroPadding2D(padding=(1, 1), data_format=None),
tf.keras.layers.Conv2D(64,(7,7), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D(5,5),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.45),
tf.keras.layers.Dense(10, activation='softmax')
 ] )<choose_model_class>
X_subset = train[train['wheezy-copper-turtle-magic'] == 1][numcols]
Y_subset = train[train['wheezy-copper-turtle-magic'] == 1]['target']
for k in range(2, 10):
knc = KNeighborsClassifier(n_neighbors=k)
 knc.fit(X_subset, Y_subset)
 score = knc.score(X_subset, Y_subset)
 print("[{}] score: {:.2f}".format(k, score))
network.compile(optimizer=Adam(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'] )<split>
filters = [('StandardScaler', StandardScaler()), 
('MinMaxScaler', MinMaxScaler()), 
('PCA', PCA(n_components=0.98)) , 
('KernelPCA(poly)', KernelPCA(kernel="poly", degree=3, gamma=15)) , 
('KernelPCA(rbf)', KernelPCA(kernel="rbf")) ,
]
for name, f in filters:
X_subset2 = f.fit_transform(X_subset)
 knc = KNeighborsClassifier(n_neighbors=3)
 knc.fit(X_subset2, Y_subset)
 score = knc.score(X_subset2, Y_subset)
 print("[{}] score: {:.2f}".format(name, score))
X_train, X_test, y_train, y_test = train_test_split(train_images, y_train, test_size=0.15 )<categorify>
knn3 = KNeighborsClassifier(n_neighbors=3)
 knnb = BaggingClassifier(base_estimator=knn3, n_estimators=20)
 knnb.fit(X_subset, Y_subset)
 score = knnb.score(X_subset, Y_subset)
 print("score: {:.2f}".format(score))
train_labels = to_categorical(y_train)
 test_labels = to_categorical(y_test)
 y_train = train_labels
y_train.shape 
<train_model>
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()
datagen = ImageDataGenerator(rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1)
 datagen.fit(X_train)
 np.concatenate(( X_train,X_train),axis=0)
 random.seed(12345)
 for X_batch, y_batch in datagen.flow(np.concatenate(( X_train,X_train),axis=0), np.concatenate(( y_train,y_train),axis=0), batch_size=35700):
break
X_trainE = X_batch
y_trainE = y_batch
for i in range(0, 9):
pyplot.subplot(330 + 1 + i)
 pyplot.imshow(X_trainE[i].reshape(28, 28), cmap=pyplot.get_cmap('gray'))
pyplot.show() 
<concatenate>
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )
X_Combine = np.concatenate(( X_train,X_trainE),axis=0)
 y_combine = np.concatenate(( y_train,y_trainE),axis=0)
 print(X_Combine.shape)
 print(y_combine.shape )<train_model>
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('val_acc')>=0.9930):
print("
Reached 99.0% accuracy so cancelling training!")
 self.model.stop_training = True
callbacks = myCallback()<train_model>
X_train = all_data[all_data['target'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['target'].isnull() ].drop(['target'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
X_train.drop(['id'], axis=1, inplace=True)
 X_test_ID = X_test.pop('id')
 Y_train = X_train.pop('target')
 print(X_train.shape, X_test.shape )
history = network.fit(X_Combine, y_combine, epochs=60, batch_size=512,validation_data=(X_test,test_labels),callbacks=[callbacks] )<compute_test_metric>
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 splits = 11
for i in tqdm(range(512)) :
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[train2.index]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = sel.fit_transform(train2)
 test2 = sel.transform(test2)
 skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train2[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] = clf.predict_proba(train2[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits
test_loss, test_acc = network.evaluate(X_test, test_labels)
 print('test_acc:', test_acc )<predict_on_test>
error = X_train[(( Y_train == 1)&(oof_preds < 0.5)) |(( Y_train == 0)&(oof_preds >= 0.5)) ]
print('error rate: %.3f%%' %(len(error)/ X_train.shape[0] * 100))
predict = network.predict(test_images_final)
 y_classes = [np.argmax(y, axis=None, out=None)for y in predict]<save_to_csv>
X_test_p1 = X_test[(sub_preds <= 0.01)].copy()
X_test_p2 = X_test[(sub_preds >= 0.99)].copy()
X_test_p1['target'] = 0
X_test_p2['target'] = 1
print(X_test_p1.shape, X_test_p2.shape)
 Y_train = pd.concat([Y_train, X_test_p1.pop('target'), X_test_p2.pop('target')], axis=0)
 X_train = pd.concat([X_train, X_test_p1, X_test_p2], axis=0)
 Y_train.reset_index(drop=True, inplace=True)
 X_train.reset_index(drop=True, inplace=True )
x = list(range(1, 28001))
df = pd.DataFrame({'ImageId' :x,'Label':y_classes})
 df.to_csv("output.csv",index=False)
<import_modules>
_=
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers<categorify>
splits = 11
Y_train_org = Y_train.copy()
X_train_org = X_train.copy()
for itr in range(4):
X_test_p1 = X_test[(sub_preds <= 0.04)].copy()
X_test_p2 = X_test[(sub_preds >= 0.96)].copy()
X_test_p1['target'] = 0
X_test_p2['target'] = 1
print(X_test_p1.shape, X_test_p2.shape)
 Y_train = pd.concat([Y_train_org, X_test_p1.pop('target'), X_test_p2.pop('target')], axis=0)
 X_train = pd.concat([X_train_org, X_test_p1, X_test_p2], axis=0)
 Y_train.reset_index(drop=True, inplace=True)
 X_train.reset_index(drop=True, inplace=True)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for i in tqdm(range(512)) :
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[train2.index]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = pd.DataFrame(sel.fit_transform(train2))
test2 = pd.DataFrame(sel.transform(test2))
sel = StandardScaler()
train2 = pd.DataFrame(sel.fit_transform(train2))
test2 = pd.DataFrame(sel.transform(test2))
skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=42, nu=0.59, coef0=0.053)
 clf.fit(train2.iloc[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] += clf.predict_proba(train2.iloc[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 return np.array(all_tokens )<choose_model_class>
submission = pd.DataFrame({
'id': X_test_ID,
'target': sub_preds
})
 submission.to_csv("submission.csv", index=False )