Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<concatenate>
tqdm.pandas()
given_dataset = pd.concat([given_train.drop('label', axis=1), given_test] ).reset_index()
true_mnist = pd.concat([true_train, true_test] ).reset_index(drop=True)
 labels = true_mnist['label'].values
true_mnist.drop('label', axis=1, inplace=True)
 true_mnist.columns = cols<sort_values>
embedding_size = 300
max_features = 95000
max_len = 100
true_idx = true_mnist.sort_values(by=list(true_mnist.columns)).index
dataset_from = given_dataset.sort_values(by=list(true_mnist.columns)) ['dataset'].values
original_idx = given_dataset.sort_values(by=list(true_mnist.columns)) ['index'].values<feature_engineering>
def clean_text(x):
x = str(x)
 for p in string.punctuation + '“…':
x = x.replace(p, ' ' + p + ' ')
 x = x.replace('_', '')
 x = re.sub("`","'", x)
 x = re.sub("(?i)n't",' not', x)
 x = re.sub("(?i)'re",' are', x)
 x = re.sub("(?i)'s",' is', x)
 x = re.sub("(?i)'d",' would', x)
 x = re.sub("(?i)'ll",' will', x)
 x = re.sub("(?i)'t",' not', x)
 x = re.sub("(?i)'ve",' have', x)
 x = re.sub("(?i)'m",' am', x)
 x = re.sub("(?i)n\’t",' not', x)
 x = re.sub("(?i)\’re",' are', x)
 x = re.sub("(?i)\’s",' is', x)
 x = re.sub("(?i)\’d",' would', x)
 x = re.sub("(?i)\’ll",' will', x)
 x = re.sub("(?i)\’t",' not', x)
 x = re.sub("(?i)\’ve",' have', x)
 x = re.sub("(?i)\’m",' am', x)
 x = re.sub('(?i)Quorans','Quora', x)
 x = re.sub('(?i)Qoura','Quora', x)
 x = re.sub('(?i)Quoran','Quora', x)
 x = re.sub('(?i)dropshipping','drop shipping', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 x = re.sub('(?i)WooCommerce','Commerce', x)
 x = re.sub('(?i)hairfall','hair fall', x)
 x = re.sub('(?i)marksheet','mark sheet', x)
 x = re.sub('(?i)articleship','article ship', x)
 x = re.sub('(?i)cryptocurrencies','cryptocurrency', x)
 x = re.sub('(?i)coinbase','cryptocurrency', x)
 x = re.sub('(?i)altcoin','bitcoin', x)
 x = re.sub('(?i)altcoins','bitcoins', x)
 x = re.sub('(?i)litecoin','bitcoin', x)
 x = re.sub('(?i)litecoins','bitcoins', x)
 x = re.sub('(?i)demonetisation','demonetization', x)
 x = re.sub('(?i)ethereum','bitcoin', x)
 x = re.sub('(?i)ethereums','bitcoins', x)
 x = re.sub('(?i)quorans','quora', x)
 x = re.sub('(?i)Brexit','britan exit', x)
 x = re.sub('(?i)upwork','freelance', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)Blockchain','blockchain', x)
 x = re.sub('(?i)GDPR','General Data Protection Regulation', x)
 x = re.sub('(?i)Qoura','quora', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Cryptocurrency','cryptocurrency', x)
 x = re.sub('(?i)Binance','cryptocurrency', x)
 x = re.sub('(?i)Redmi','mobile phone', x)
 x = re.sub('(?i)TensorFlow','Tensor Flow', x)
 x = re.sub('(?i)Golang','programming language', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 return x
for i in range(len(true_idx)) :
if dataset_from[i] == 'test':
sample_submission.loc[original_idx[i], 'Label'] = labels[true_idx[i]]<save_to_csv>
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32' )
sample_submission.to_csv('submission.csv', index=False )<set_options>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
%matplotlib inline 
<load_from_csv>
def get_model_1(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(80, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Conv1D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform' )(x)
 y = Dense(16, activation='relu' )(y)
 y = Flatten()(y)
 atten = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([atten, avg_pool, max_pool])
 conc = Dense(64, activation='relu' )(conc)
 conc_z = concatenate([conc, y])
 conc_z = Dense(32, activation='relu' )(conc_z)
 conc_z = Dropout(0.1 )(conc_z)
 output = Dense(1, activation='sigmoid' )(conc_z)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train.head()
print(train['Embarked'].unique() , train['Pclass'].unique())
 train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
<drop_column>
def get_model_2(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(max_len )(x)
 output = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
for df in [train, test]:
df.drop(labels=["PassengerId", "Cabin", "Name", "Ticket"], axis=1, inplace=True )<feature_engineering>
def get_model_3(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 42
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Reshape(( max_len, embedding_size, 1))(x)
 conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 maxpool_0 = MaxPool2D(pool_size=(max_len - filter_sizes[0] + 1, 1))(conv_0)
 maxpool_1 = MaxPool2D(pool_size=(max_len - filter_sizes[1] + 1, 1))(conv_1)
 maxpool_2 = MaxPool2D(pool_size=(max_len - filter_sizes[2] + 1, 1))(conv_2)
 maxpool_3 = MaxPool2D(pool_size=(max_len - filter_sizes[3] + 1, 1))(conv_3)
 z = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 output = Dense(1, activation='sigmoid' )(z)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
for df in [train, test]:
for col in ["Age", "Fare"]:
df[col] = df[col].fillna(np.mean(df[col]))<normalization>
def get_model_4(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
 atten = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([atten, avg_pool, max_pool])
 conc = Dense(64, activation='relu' )(conc)
 conc = Dropout(0.1 )(conc)
 output = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
min_max_scaler = preprocessing.MinMaxScaler()
for df in [train, test]:
for col in ["Age", "Fare"]:
x = df[[col]].values.astype(float)
 df[col] = min_max_scaler.fit_transform(x )<feature_engineering>
def get_model_5(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(max_len )(x)
 x = Dense(16, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 output = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
for df in [train, test]:
df['is_male'] = np.where(df['Sex']=="male", 1, 0)
 df['is_female'] = np.where(df['Sex']=="female", 1, 0)
 df['EmbarkedS'] = np.where(df['Embarked']=="S", 1, 0)
 df['EmbarkedC'] = np.where(df['Embarked']=="C", 1, 0)
 df['EmbarkedQ'] = np.where(df['Embarked']=="Q", 1, 0)
 df['Pclass1'] = np.where(df['Pclass']==1, 1, 0)
 df['Pclass2'] = np.where(df['Pclass']==2, 1, 0)
 df['Pclass3'] = np.where(df['Pclass']==3, 1, 0)
 df['is_single'] = np.where(np.logical_and(df['SibSp']==0, df['Parch']==0), 1, 0)
 for df in [train, test]:
df.drop(labels=["Sex", "Embarked", 'Pclass'], axis=1, inplace=True )<prepare_x_and_y>
def get_model_6(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.15 )(x)
 x = Bidirectional(CuDNNLSTM(max_len, return_sequences=True))(x)
 x = Conv1D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform' )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 output = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
train_size = int(train.shape[0] * 0.85)
 train_dataset = train[:train_size]
val_dataset = train[train_size:]
X_train = train_dataset.drop(labels=["Survived"], axis=1 ).values
Y_train = train_dataset["Survived"].values
X_val = val_dataset.drop(labels=["Survived"], axis=1 ).values
Y_val = val_dataset["Survived"].values
input_size = len(train_dataset.columns)- 1<choose_model_class>
def get_f1_metric(y_true, y_predicted):
f1_metrics = []
for thresh in np.arange(0.1, 0.51, 0.01):
thresh = np.round(thresh, 2)
 f1_metric = f1_score(y_true,(y_predicted>thresh ).astype(int))
f1_metrics.append(( thresh, f1_metric))
print('F1 score at threshold {0} is {1}'.format(thresh, f1_metric))
threshold = max(f1_metrics, key=lambda x: x[1])[0]
print(threshold)
 return threshold
model = Sequential()
k_init = 'glorot_uniform'
optimizer = optimizers.Adam()
model.add(Dense(64,input_dim=input_size, kernel_initializer=k_init))
model.add(Activation("relu"))
model.add(Dropout(0.3))
model.add(Dense(64, kernel_initializer=k_init))
model.add(Activation("relu"))
model.add(Dropout(0.3))
model.add(Dense(1, kernel_initializer=k_init))
model.add(Activation("sigmoid"))
model.compile(loss=losses.binary_crossentropy, optimizer=optimizer,
metrics=[metrics.binary_accuracy])
 history = model.fit(X_train, Y_train, validation_data=(X_val, Y_val),
epochs=200, batch_size=100,
verbose=0)
<load_from_csv>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print('Train shape : ', train_df.shape)
 print('Test shape : ', test_df.shape )
y_final = model.predict_classes(test.values ).reshape(-1)
 df_test = pd.read_csv(".. /input/test.csv")
 output = pd.DataFrame({'PassengerId': df_test['PassengerId'], 'Survived': y_final})
 surv_num = sum(output["Survived"] != 0)/ len(output)
 print(f"Survive ratio: {surv_num}" )<save_to_csv>
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))
output.to_csv('prediction-ann.csv', index=False)
 output<set_options>
train_df, val_df = train_test_split(train_df,
test_size=0.07,
random_state=42,
stratify=train_df['target'])
 X_train = train_df['question_text'].values
y_train = train_df['target'].values
X_val = val_df['question_text'].values
y_val = val_df['target'].values
X_test = test_df['question_text'].values
print(X_train.shape,
y_train.shape,
X_val.shape,
y_val.shape,
X_test.shape )
%matplotlib notebook 
<load_from_csv>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_val))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=max_len)
 X_val = pad_sequences(X_val, maxlen=max_len)
 X_test = pad_sequences(X_test, maxlen=max_len)
 word_index = tokenizer.word_index
print(X_train.shape, X_val.shape, X_test.shape, len(word_index))
dataRawTrain = pd.read_csv('.. /input/train.csv')
 dataRawTest = pd.read_csv('.. /input/test.csv')
 dataTrain = np.array(dataRawTrain.iloc[:,1:] ).astype('uint8')
 dataTrain = dataTrain.reshape(-1, 1, 28, 28)
 targetTrain = np.array(dataRawTrain.iloc[:,:1])
 zz = np.zeros([len(targetTrain), 10])
 for i, d in enumerate(targetTrain):
zz[i][d.squeeze() ] = 0.66
targetTrain = zz
dataTest = np.array(dataRawTest.iloc[:,:] ).astype('uint8')
 dataTest = dataTest.reshape(-1, 1, 28, 28)
 for i in range(1):
plt.matshow(dataTrain[i][0], cmap='rainbow')
 plt.suptitle('class: {}'.format(targetTrain[i]))
for i in range(1):
plt.matshow(dataTest[i][0], cmap='rainbow')
 plt.show() 
<choose_model_class>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix_glove = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_glove[i] = embedding_vector
del embeddings_index
gc.collect()
print(embedding_matrix_glove.shape )
class NetMNIST2(nn.Module):
flag = False
def __init__(self):
super(NetMNIST2, self ).__init__()
self.conv0 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1, bias=True)
 self.relu0 = nn.PReLU()
self.pool0 = nn.MaxPool2d(2)
 self.drop0 = nn.Dropout2d()
self.conv1p = nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1, bias=False)
 self.relu1p = nn.PReLU()
self.drop1p = nn.Dropout2d()
self.conv1 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1, bias=True)
 self.relu1 = nn.PReLU()
self.pool1 = nn.MaxPool2d(2)
 self.drop1 = nn.Dropout2d()
self.conv2 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1, bias=False)
 self.relu2 = nn.PReLU()
self.pool2 = nn.MaxPool2d(2)
 self.drop2 = nn.Dropout2d()
self.fc1 = nn.Linear(32 * 6 * 6, 10)
 self.fc1Prelu = nn.PReLU()
self.fc2 = nn.Linear(10, 10)
 def forward(self, x):
x = self.conv0(x)
 x = self.relu0(x)
 x = self.pool0(x)
 x = self.drop0(x)
 x = self.conv1p(x)
 x = self.relu1p(x)
 x = self.drop1p(x)
 x = self.conv1(x)
 x = self.relu1(x)
 x = self.pool1(x)
 x = self.drop1(x)
 x = self.conv2(x)
 x = self.relu2(x)
 x = self.pool2(x)
 x = self.drop2(x)
 x = x.view(-1, 128*3*3)
 x = self.fc1(x)
 x = self.fc1Prelu(x)
 x = torch.nn.functional.dropout(x)
 x = self.fc2(x)
 x = F.softmax(x, dim=1)
 return x 
<train_model>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_wiki[i] = embedding_vector
del embeddings_index, word_index
gc.collect()
print(embedding_matrix_wiki.shape )
use_cuda = True
device = torch.device("cuda" if use_cuda else "cpu")
 net = NetMNIST2().to(device)
 print(net)
 optimizer = optim.Adam(net.parameters() , lr=1e-4, weight_decay=1e-5)
 criterion = F.mse_loss
def trainEpoch(e):
net.train()
for i,(data, target)in enumerate(loaderTrain):
dataCUDA, targetCUDA = Variable(data ).to(device), Variable(target.float() ).to(device)
 optimizer.zero_grad()
outModel = net(dataCUDA)
 loss = F.mse_loss(outModel, targetCUDA)
 loss.backward()
optimizer.step()
return loss
def testLoss() :
net.eval()
fit = 0.0
for i,(data, target)in enumerate(loaderTest):
dataCUDA, targetCUDA = Variable(data ).to(device), Variable(target.float() ).to(device)
 outModel = net(dataCUDA)
 pred = outModel.data.max(1)[1]
targetMaxIndex = targetCUDA.data.max(1)[1]
fit += pred.eq(targetMaxIndex ).cpu().sum()
acc = float(fit.cpu())/ float(len(loaderTest.dataset))
return acc
lossProgress = []
for e in range(5):
lossTrain = trainEpoch(e)
 accTest = testLoss()
lossProgress.append(accTest)
 print('Test epoch: {} acc: {}'.format(e, accTest))
print('Train epoch: {} loss: {}'.format(e, lossTrain))
plt.plot(lossProgress)
<save_to_csv>
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_wiki], axis = 0)
 np.shape(embedding_matrix )
packSize = 10
inputLen = dataTestT.size(0)
 indexInput = np.arange(1, inputLen + 1)
 result = np.zeros([inputLen],dtype=int)
 print(inputLen / packSize)
 for i in range(int(inputLen / packSize)) :
dataCUDA = Variable(dataTestT[i * packSize:i*packSize+packSize].cuda())
 outModel = net(dataCUDA)
 result[i * packSize:i*packSize+packSize] = outModel.data.max(1)[1].cpu().numpy().squeeze()
result=pd.DataFrame({'ImageId':indexInput, 'Label':result})
 result.to_csv("submission.csv",index=False)
<set_options>
model_1 = get_model_1(embedding_matrix_glove)
 model_1.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))
plt.style.use('ggplot')
<load_from_csv>
model_2 = get_model_2(embedding_matrix_glove)
 model_2.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))
train_df = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_df = pd.read_csv(".. /input/digit-recognizer/test.csv" )<data_type_conversions>
model_3 = get_model_3(embedding_matrix)
 model_3.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))
X =(train_df.iloc[:,1:].values ).astype('float32')
 Y = train_df.iloc[:,0].values.astype('int32')
 test = test_df.values.astype('float32' )<categorify>
model_4 = get_model_4(embedding_matrix)
 model_4.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))
X = X.reshape(X.shape[0], 28, 28, 1)
 Y = to_categorical(Y)
 test = test.reshape(test.shape[0], 28, 28, 1 )<data_type_conversions>
model_5 = get_model_5(embedding_matrix_wiki)
 model_5.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))
X = X.astype("float32")/ 255
test = test.astype("float32")/ 255<split>
model_6 = get_model_6(embedding_matrix_wiki)
 model_6.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size = 0.1, random_state=0 )<init_hyperparams>
pred_val_1 = model_1.predict([X_val], batch_size=512, verbose=1)
 pred_val_2 = model_2.predict([X_val], batch_size=512, verbose=1)
 pred_val_3 = model_3.predict([X_val], batch_size=512, verbose=1)
 pred_val_4 = model_4.predict([X_val], batch_size=512, verbose=1)
 pred_val_5 = model_5.predict([X_val], batch_size=512, verbose=1)
 pred_val_6 = model_6.predict([X_val], batch_size=512, verbose=1 )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
pred_val_1_df = np.reshape(pred_val_1,(pred_val_1.shape[0]))
pred_val_2_df = np.reshape(pred_val_2,(pred_val_2.shape[0]))
pred_val_3_df = np.reshape(pred_val_3,(pred_val_3.shape[0]))
pred_val_4_df = np.reshape(pred_val_4,(pred_val_4.shape[0]))
pred_val_5_df = np.reshape(pred_val_5,(pred_val_5.shape[0]))
pred_val_6_df = np.reshape(pred_val_6,(pred_val_6.shape[0]))
pred_val_df = np.reshape(y_val,(y_val.shape[0]))
validation_df = pd.DataFrame({'val_1': pred_val_1_df, 'val_2': pred_val_2_df, 'val_3': pred_val_3_df, 'val_4': pred_val_4_df, 'val_5': pred_val_5_df, 'val_6': pred_val_6_df, 'prediction': pred_val_df})
 validation_df.to_csv('validation.csv', index=False )
model = Sequential()
model.add(Conv2D(32, kernel_size = 4, activation="relu", input_shape=(28,28,1)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64, kernel_size = 3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64, kernel_size = 2, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(128, activation="relu"))
model.add(Dropout(0.25))
model.add(Dense(10, activation='softmax'))
model.compile(loss="mean_squared_error", optimizer="rmsprop", metrics=['accuracy'])
 model.summary()<choose_model_class>
validation_df['val_12_mean'] =(validation_df['val_1'] + validation_df['val_2'])/ 2.0
validation_df['val_34_mean'] =(validation_df['val_3'] + validation_df['val_4'])/ 2.0
validation_df['val_56_mean'] =(validation_df['val_5'] + validation_df['val_6'])/ 2.0
validation_df['val_123_mean'] =(validation_df['val_1'] + validation_df['val_2'] + validation_df['val_3'])/ 3.0
validation_df['val_456_mean'] =(validation_df['val_4'] + validation_df['val_5'] + validation_df['val_6'])/ 3.0
validation_df['val_1_log'] = np.log(validation_df['val_1'])
 validation_df['val_2_log'] = np.log(validation_df['val_2'])
 validation_df['val_3_log'] = np.log(validation_df['val_3'])
 validation_df['val_4_log'] = np.log(validation_df['val_4'])
 validation_df['val_5_log'] = np.log(validation_df['val_5'])
 validation_df['val_6_log'] = np.log(validation_df['val_6'])
 validation_df['val_12_log_mean'] =(validation_df['val_1_log'] + validation_df['val_2_log'])/ 2.0
validation_df['val_34_log_mean'] =(validation_df['val_3_log'] + validation_df['val_4_log'])/ 2.0
validation_df['val_56_log_mean'] =(validation_df['val_5_log'] + validation_df['val_6_log'])/ 2.0
validation_df['val_123_log_mean'] =(validation_df['val_1_log'] + validation_df['val_2_log'] + validation_df['val_3_log'])/ 3.0
validation_df['val_456_log_mean'] =(validation_df['val_4_log'] + validation_df['val_5_log'] + validation_df['val_6_log'])/ 3.0
validation_df = validation_df[[c for c in validation_df.columns if c != 'prediction'] + ['prediction']]
validation_df.head()
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
X_val = validation_df[validation_df.columns.values[:-1]].values
y_val = validation_df[validation_df.columns.values[-1]].values
clf = VotingClassifier(estimators=
[('gnb', GaussianNB()), 
('xgb', xgb.XGBClassifier(max_dept=100, n_estimators=15, learning_rate=0.05,
colsample_bytree=0.5, gamma=0.01, reg_alpha=4,
objective='binary:logistic')) , 
('knn', KNeighborsClassifier(n_neighbors=500)) , 
('rf', RandomForestClassifier(max_depth=3, n_estimators=100)) , 
('lr', LogisticRegression()), 
('dt', DecisionTreeClassifier(max_depth=3, criterion='entropy')) , 
('adb', AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3, criterion='entropy'), learning_rate=0.05)) , 
('gb', GradientBoostingClassifier(max_depth=3, learning_rate=0.05,
n_estimators=100)) , 
('qda', QuadraticDiscriminantAnalysis()), 
('lda', LinearDiscriminantAnalysis())
 ],
voting='soft')
 clf.fit(X_val, y_val )
batch_size = 100
epochs = 30
hist = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<prepare_output>
pred_test_1 = model_1.predict([X_test], batch_size=512, verbose=1)
 pred_test_2 = model_2.predict([X_test], batch_size=512, verbose=1)
 pred_test_3 = model_3.predict([X_test], batch_size=512, verbose=1)
 pred_test_4 = model_4.predict([X_test], batch_size=512, verbose=1)
 pred_test_5 = model_5.predict([X_test], batch_size=512, verbose=1)
 pred_test_6 = model_6.predict([X_test], batch_size=512, verbose=1 )
Y_pred_classes = np.argmax(pred,axis = 1)
 Y_true = np.argmax(Y_val,axis = 1 )<predict_on_test>
pred_test_1_df = np.reshape(pred_test_1,(pred_test_1.shape[0]))
pred_test_2_df = np.reshape(pred_test_2,(pred_test_2.shape[0]))
pred_test_3_df = np.reshape(pred_test_3,(pred_test_3.shape[0]))
pred_test_4_df = np.reshape(pred_test_4,(pred_test_4.shape[0]))
pred_test_5_df = np.reshape(pred_test_5,(pred_test_5.shape[0]))
pred_test_6_df = np.reshape(pred_test_6,(pred_test_6.shape[0]))
test_df1 = pd.DataFrame({'val_1': pred_test_1_df, 'val_2': pred_test_2_df, 'val_3': pred_test_3_df, 'val_4': pred_test_4_df, 'val_5': pred_test_5_df, 'val_6': pred_test_6_df})
 test_df1.to_csv('test.csv', index=False )
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
test_df1['val_12_mean'] =(test_df1['val_1'] + test_df1['val_2'])/ 2.0
test_df1['val_34_mean'] =(test_df1['val_3'] + test_df1['val_4'])/ 2.0
test_df1['val_56_mean'] =(test_df1['val_5'] + test_df1['val_6'])/ 2.0
test_df1['val_123_mean'] =(test_df1['val_1'] + test_df1['val_2'] + test_df1['val_3'])/ 3.0
test_df1['val_456_mean'] =(test_df1['val_4'] + test_df1['val_5'] + test_df1['val_6'])/ 3.0
test_df1['val_1_log'] = np.log(test_df1['val_1'])
 test_df1['val_2_log'] = np.log(test_df1['val_2'])
 test_df1['val_3_log'] = np.log(test_df1['val_3'])
 test_df1['val_4_log'] = np.log(test_df1['val_4'])
 test_df1['val_5_log'] = np.log(test_df1['val_5'])
 test_df1['val_6_log'] = np.log(test_df1['val_6'])
 test_df1['val_12_log_mean'] =(test_df1['val_1_log'] + test_df1['val_2_log'])/ 2.0
test_df1['val_34_log_mean'] =(test_df1['val_3_log'] + test_df1['val_4_log'])/ 2.0
test_df1['val_56_log_mean'] =(test_df1['val_5_log'] + test_df1['val_6_log'])/ 2.0
test_df1['val_123_log_mean'] =(test_df1['val_1_log'] + test_df1['val_2_log'] + test_df1['val_3_log'])/ 3.0
test_df1['val_456_log_mean'] =(test_df1['val_4_log'] + test_df1['val_5_log'] + test_df1['val_6_log'])/ 3.0
test_df1.head()
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<set_options>
X_test = test_df1.values
y_test_pred = clf.predict(X_test)
 y_test_pred = np.reshape(y_test_pred,(y_test_pred.shape[0]))
<load_from_csv><EOS> 
submission_df = pd.DataFrame({'qid': test_df['qid'].values, 'prediction': y_test_pred})
 submission_df.to_csv('submission.csv', index=False )