Kaggle competition: Characters classification
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=3)
 outputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'] )<compute_test_metric>
import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import np_utils
from skimage.transform import rescale
from sklearn import preprocessing
from keras.models import load_model
from keras.layers import BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from sklearn.model_selection import train_test_split
from tqdm import tqdm
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]<compute_test_metric>
img_rows = 40
img_cols = 40
print("Best threshold:", best_thresh, "and F1 score", thresholds[0][1] )<data_type_conversions>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int )<save_to_csv>
encoder = preprocessing.LabelEncoder()
encoder.fit(np.load(".. /input/tmpfory/trainY.npy"))
sub = pd.read_csv('.. /input/sample_submission.csv')
 out_df = pd.DataFrame({"qid":sub["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
T = np.load(".. /input/inputtestdata40x40/testT.npy" )
tqdm.pandas(desc='Progress')
<define_variables>
model = Sequential()
model.add(Conv2D(64, kernel_size = 3, activation='relu', input_shape =(img_rows, img_cols, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(256, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(1000, activation='softmax'))
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
 print(model.summary() )
embed_size = 400
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 8
SEED = 229<set_options>
datagen = ImageDataGenerator(
 rotation_range=8,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )
def seed_everything(seed=SEED):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<statistical_test>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_from_csv>
X_train2 = np.load(".. /input/rescalinginputdata/X_train2.npy")
 X_val2 = np.load(".. /input/rescalinginputdata/X_val2.npy")
 Y_train2 = np_utils.to_categorical(encoder.transform(np.load(".. /input/rescalinginputdata/Y_train2.npy")))
 Y_val2 = np_utils.to_categorical(encoder.transform(np.load(".. /input/rescalinginputdata/Y_val2.npy")) )
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df = pd.concat([df_train ,df_test],sort=True )<feature_engineering>
history = model.fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),
epochs = 18, steps_per_epoch = X_train2.shape[0]//64,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0, initial_epoch=0 )
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = build_vocab(df['question_text'] )<define_variables>
history = history.history
file = open('info.csv', 'w')
 file.write(str(history))
file.close()
sin = len(df_train[df_train["target"]==0])
 insin = len(df_train[df_train["target"]==1])
 persin =(sin/(sin+insin)) *100
perinsin =(insin/(sin+insin)) *100
print("
print("<categorify>
predictions = model.predict(T)
 predictions = np.argmax(predictions, axis=1)
 res = encoder.inverse_transform(predictions )
def get_wired(d):
wired = []
for v in d:
m = re.match('[\w{}]+', v)
 if not m or m[0] != v:
wired += [v]
return wired
wired = get_wired(vocab)
 len(wired )<define_variables>
file = open('ansEpochs.csv', 'w')
 file.write('Id,Category
')
 for i in range(res.size):
file.write('' + str(i + 1)+ ',' + str(res[i])+ '
')
 file.close()