Kaggle competition: HTA Tagging
<SOS> metric: categorizationaccuracy Kaggle data source: hta-tagging<statistical_test>
seed = 300
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
from math import *
import math
import numpy as np
import pandas as pd
print('Wiki...')
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_wiki.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<feature_engineering>
Train = pd.read_csv(".. /input/hta-tagging/train.csv")
 Train.head(5 )
matrixes = [embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()<import_modules>
Test = pd.read_csv(".. /input/hta-tagging/test.csv")
 Test.head(5 )
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
def _threshold_search(self,y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def on_epoch_end(self, epoch, logs={}):
y_val = self.validation_data[1]
y_pred = self.model.predict(self.validation_data[0])
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search_result = self._threshold_search(y_val, y_pred)
 _val_f1 = search_result['f1']
self.val_f1s.append(_val_f1)
 print('— val_f1_on_epoch_end: %f '%(_val_f1))
return
def get_model(adam, trainable=False):
inp1 = Input(shape=(max_len,))
emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 filters = []
for f in [3,4,5]:
conv = Conv1D(64,f, activation='elu' )(emb)
 pool = GlobalMaxPooling1D()(conv)
 filters.append(pool)
 x = concatenate(filters)
 x = Dense(max_len, activation="relu" )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
 print(model.summary())
 return model
def get_opt() :
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
return adam
def get_batch_size(opt):
batch_size = int(x_train.shape[0]*K.eval(opt.lr))
print('Batch size = ',batch_size)
 return batch_size
patience = 2
best_model=None
all_results = {}
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('Pretraining finished, unfreezing embeddings layer...')
 model = get_model(adam,True)
 model.load_weights(model_file)
 history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('training finished...')
 model.load_weights(model_file)
 y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_val, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON VALIDATION SET:
',classification_report(y_val,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))
all_results[model_file] = search_result['f1']
if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
<predict_on_test>
def ReadFileToDataFrame(df,path):
for i in range(len(df.Filename)) :
Filename = df.Filename[i];
CurrentFile = ".. /input/hta-tagging/{2}/{2}/{0}/{1}".format(Filename.split("-")[0],Filename,path);
File = open(CurrentFile, "r");
contents = File.read() ;
contents = contents.replace("\r
","
")
 contents = contents.replace("\r","
")
 df.Filename[i] = contents
continue;
df = df.rename(columns={"Filename":"Text"},inplace=True);
ReadFileToDataFrame(Train,"train-data");
Train.head(5 )
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 model.load_weights(best_model['model'])
 y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))<save_to_csv>
def ReadTestFileToDataFrame(df,path):
df["Text"] = [""]*len(df.Id);
for i in range(len(df.Id)) :
Filename = df.Id[i];
CurrentFile = ".. /input/hta-tagging/{2}/{2}/{0}/{1}".format(Filename.split("-")[0],Filename,path);
def Do(File):
contents = File.read() ;
contents = contents.replace("\r
","
")
 contents = contents.replace("\r","
")
 df.Text[i] = contents
try:
with codecs.open(CurrentFile, encoding='cp1252')as File:
Do(File);
except:
with codecs.open(CurrentFile, encoding='utf-8')as File:
Do(File);
continue;
df = df.rename(columns={"Filename":"Text"},inplace=True);
ReadTestFileToDataFrame(Test,"test-data");
Test.head(5 )
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_final=tokenizer.texts_to_sequences(df_final['question_text'])
 x_final = pad_sequences(x_final,maxlen=max_len)
 y_pred = model.predict(x_final,batch_size=batch_size,verbose=1)
 y_pred = y_pred > search_result['threshold']
y_pred = y_pred.astype(int)
 print(y_pred[:5])
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = y_pred
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<string_transform>
def AnsEncoder(df, col_name):
for i in range(len(df[col_name])) :
val = 0;
txt = df.loc[i,col_name];
if(txt == "P"): val = 2;
if(txt == "Q"): val = 1;
if(txt == "N"): val = 0;
df.loc[i,col_name] = val;
continue;
df[col_name] = pd.to_numeric(df[col_name]);
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('spliting data')
 df_train,df_test = train_test_split(df, random_state=1)
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df_train['question_text'])
 x_test = tokenizer.texts_to_sequences(df_test['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 x_test = pad_sequences(x_test, maxlen=max_len)
 y_train = df_train['target'].values
y_test = df_test['target'].values
print(x_train.shape)
 print(y_train.shape )<compute_train_metric>
AnsEncoder(Train, "Blinding of intervention");
AnsEncoder(Train, "Blinding of Outcome assessment");
Train.head(5 )
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
def remove_ref(text):
newtext = "";
for line in text.split("
"):
if(line.lower() =="references" or line.lower() =="reference"): break;
else: newtext += line + "
";
return text;
Train.Text = Train.Text.apply(lambda x : remove_ref(x))
Test.Text = Test.Text.apply(lambda x : remove_ref(x))
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
def FindBeforeColon(text):
def kwreplace(cutted):
cutted = cutted.lower() ;
if("key" in cutted): cutted = "key";
elif("objective" in cutted): cutted = "objective";
elif("reference" in cutted): cutted = "reference";
elif("method" in cutted): cutted = "method";
elif(cutted.endswith("es")) : cutted = cutted[:-2];
elif(cutted.endswith("s")) : cutted = cutted[:-1];
return cutted;
tmptxt = "";
curtopic = "";
topic = [];
for line in text.split("
"):
if(":" in line):
cutted = line.split(":")[0];
if(len(cutted)>0):
if(len(cutted.split(" ")) < 6 and cutted[0] in "ABCDEFGHIJKLMNOPQRSTUVWXYZ" and(not "Author" in cutted)) :
cutted = kwreplace(cutted);
if(curtopic != ""):
topic.append(( curtopic, tmptxt)) ;
tmptxt = "";
curtopic = cutted;
tmptxt = "".join(line.split(":")[1:]);
continue;
if(len(line)>0):
if(not " " in line and line[0] in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
if(curtopic != ""):
topic.append(( curtopic, tmptxt)) ;
curtopic = kwreplace(line);
tmptxt = "";
continue;
tmptxt += "
" + line;
return topic;
print('Wiki...')
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_wiki.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<feature_engineering>
TO = [];
i = 0;
for txt in Train.Text:
TO.append(( i,FindBeforeColon(txt)));
i += 1
TestO = [];
i = 0;
for txt in Test.Text:
TestO.append(( i,FindBeforeColon(txt)));
i += 1;
matrixes = [embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()<import_modules>
SelectedCols = ["Text","method","key"]
def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def get_model(adam):
inp1 = Input(shape=(max_len,))
x = Embedding(num_words, dim, weights=[matrix],trainable = False, )(inp1)
 x = Bidirectional(CuDNNLSTM(int(max_len/2),return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool,max_pool])
 x = Dense(max_len, activation="relu" )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy',f1])
 print(model.summary())
 return model
def get_opt() :
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
return adam
def get_batch_size(opt):
batch_size = int(x_train.shape[0]*K.eval(opt.lr))
print('Batch size = ',batch_size)
 return batch_size
patience = 2
best_model=None
all_results = {}
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 x_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
x_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 history = model.fit(x_train1,y_train1,
batch_size=batch_size,
validation_data=(x_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce],
verbose=2)
 print('training finished...')
 model.load_weights(model_file)
 y_pred = model.predict(x_train1,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_train1, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TRAINING SET:
',classification_report(y_train1,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))
all_results[model_file] = search_result['f1']
if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
<predict_on_test>
def WriteToPD(df,arr):
arrlength = len(arr);
PlaceholderList = [None]*df.shape[0];
for idx, optionlist in arr:
if(idx % 100 == 0):
print("{0}/{1} Completed".format(idx,arrlength)) ;
for option, text in optionlist:
if option in SelectedCols:
if not option in df:
df[option] = PlaceholderList;
df.loc[idx,option] = text;
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 model.load_weights(best_model['model'])
 y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))<save_to_csv>
WriteToPD(Train,TO)
 Train.head()
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_final=tokenizer.texts_to_sequences(df_final['question_text'])
 x_final = pad_sequences(x_final,maxlen=max_len)
 y_pred = model.predict(x_final,batch_size=batch_size,verbose=1)
 y_pred = y_pred > search_result['threshold']
y_pred = y_pred.astype(int)
 print(y_pred[:5])
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = y_pred
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<import_modules>
WriteToPD(Test,TestO)
 Train.head()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
class TFIDFVector:
def __init__(this,train,val):
print("- TF-IDF Vector: Initializing")
 this.tfv = TfidfVectorizer(min_df=3, max_features=None,
strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,
stop_words = 'english');
print("- TF-IDF Vector: Preparing");
if type(train[0])!= str:
tl2 = [];
for l in train:
tl2 += [txt for txt in l if txt != None];
for l in val:
tl2 += [txt for txt in l if txt != None];
print("- TF-IDF Vector: Fitting, This might take a while");
this.tfv.fit(tl2);
print("- TF-IDF Vector: Fitted");
del(tl2);
else:
print("- TF-IDF Vector: Fitting, This might take a while");
this.tfv.fit(list(train)+list(val)) ;
print("- TF-IDF Vector: Fitted");
print("- TF-IDF Vector: Initialized")
 def CreateVector(this, val, reduceshape = True):
print("- TF-IDF Vector: Creating Vector")
 if type(val[0])!= str:
ValLength = len(val);
tmplist = []; i = 0;
for ll in val:
if(i % max(ValLength//10,1)== 0): print("- TF-IDF Vector: Creating Vector {0}/{1} Completed".format(i,ValLength)) ;
tmplist.append(( this.tfv.transform(['' if txt is None else txt for txt in ll])));
i += 1;
v = np.stack(np.array([x.toarray() for x in tmplist]))
del(tmplist)
 if(reduceshape):
nsamples = v.shape[0];
value2 = np.prod(v.shape[1:])
 v = v.reshape(( nsamples,value2))
to_return = v;
else:
to_return = this.tfv.transform(val)
 print("- TF-IDF Vector: Vector Created")
 return to_return;
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
class SVMModel:
def __init__(this):
return;
def Train(this,X, Y):
print("SVMModel: Training Step will be combined with Predict State")
 this.X = X; this.Y = Y;
def SetVector(this,Vector):
this.VC = Vector;
def Predict(this,XVal):
X = this.VC.CreateVector(this.X); Y = this.Y;
del(this.X);
this.svc = SVC(random_state=seed)
 print("SVMModel: Training")
 this.svc.fit(X, Y); del(X); del(Y);
print("SVMModel: Finished Training")
 print("SVMModel: Predicting");
valv = this.VC.CreateVector(XVal);
del(XVal);
tR = this.svc.predict(valv);
del(valv);
print("SVMModel: Predicted");
return tR;
train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 70
train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<randomize_order>
def PredictModel(TX,TY,VX,Model):
print("Initializing Model")
 model = Model;
print("Training Model")
 model.Train(( TrainX),(TrainY)) ;
print("Predicting")
 global PredictedY;
PredictedY = model.Predict(VX);
return PredictedY.astype(int);
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]<statistical_test>
TrainX, ValX, TrainY, ValY = TTS(Train[SelectedCols], Train['Blinding of intervention'], test_size=0.2, random_state = seed);
_1, _2, _3, ValY2 = TTS(Train[SelectedCols], Train['Blinding of Outcome assessment'], test_size=0.2, random_state = seed);
TrainX = np.array(TrainX);
TrainY = np.array(TrainY);
ValX = np.array(ValX);
ValY = np.array(ValY);
ValY2 = np.array(ValY2);
TrainY2 = Train['Blinding of Outcome assessment']; TrainY2 = np.array(TrainY2);
TestX = Test[SelectedCols]; TestX = np.array(TestX);
Vector = TFIDFVector(np.array(Train[SelectedCols]), TestX);
ValX = Vector.CreateVector(ValX)
 SVM = SVMModel() ;
SVM.SetVector(Vector);
PredA = PredictModel(TrainX, TrainY , TestX, SVM);
VPredA = SVM.svc.predict(ValX);
AccA = ScoreAcc(ValY,VPredA)
 SVM = SVMModel() ;
SVM.SetVector(Vector);
PredB = PredictModel(TrainX, TrainY2, TestX, SVM);
VPredB = SVM.svc.predict(ValX)
 AccB = ScoreAcc(ValY2,VPredB);
def Convert(i):
if i == 0: return "N";
if i == 1: return "Q";
if i == 2: return "P";
FinalPred = [Convert(PredA[i])+Convert(PredB[i])for i in range(len(PredA)) ];
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
Test.Prediction = FinalPred;
SubmissionTest = Test[["Id","Prediction"]];
SubmissionTest.to_csv("submission.csv",index=False)
 SubmissionTest.head(5);
<train_model><EOS> 
def Convert(i):
if i == 0: return "N";
if i == 1: return "Q";
if i == 2: return "P";
FinalPred = [Convert(VPredA[i])+Convert(VPredB[i])for i in range(len(VPredA)) ];
CorrectAns = [Convert(ValY[i])+Convert(ValY2[i])for i in range(len(ValY)) ];
FinAcc = ScoreAcc(CorrectAns,FinalPred)*100;
print("A set = {0:.2f}%".format(AccA))
print("B set = {0:.2f}%".format(AccB))
print("คะแนนที่คาดว่าจะได้ = {0:.2f}%".format(FinAcc))