Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<features_selection>
plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
stop = set(list(stop)+ ["http","https", "s", "nt", "m"])
important_feature_index = np.argsort(model3.feature_importances_)[::-1]
X_train.columns[important_feature_index[:10]]<create_dataframe>
def load_training(training_path="/kaggle/input/nlp-getting-started/train.csv"):
df = pd.read_csv(training_path)
 print(df.head(10))
return df
df = load_training()
X_train = X_train_select.copy()
X_test = X_test_select.copy()<split>
print("-Hashtag Analysis ")
 def find_hashtags(tweet):
return ", ".join([match.group(0)[1:] for match in re.finditer(r"
def add_hashtags(df):
df['hashtag'] = df["text"].apply(lambda x: find_hashtags(x))
df['hashtag'].fillna(value="no", inplace=True)
 return df
top_N = 20
df = add_hashtags(df)
 _l = len([v for v in df.hashtag.values if isinstance(v, str)])
 print("-Number of tweets with hashtags: {}".format(_l))
print("-- Hashtag distribution in positive samples ")
 show_word_distrib(target=1, field="hashtag")
 print("-- Hashtag distribution in negative samples ")
 show_word_distrib(target=0, field="hashtag")
X_bsp_train = X_train[1200:]
X_bsp_cv = X_train[:1200]
y_bsp_train = y_train[1200:]
y_bsp_cv = y_train[:1200]
y_bsp_train_new = y_train_new[1200:]
y_bsp_cv_new = y_train_new[:1200]<define_variables>
df['keyword'] = df['keyword'].map(lambda s: s.replace('%20', ' ')if isinstance(s, str)else s)
 un_KW = {kw for kw in df['keyword' ].values if isinstance(kw, str)}
tot_KW = len(df)- len(df[df["keyword" ].isna() ])
 un_LOC = {lc for lc in df['location'].values if isinstance(lc, str)}
tot_LOC = len(df)- len(df[df["location"].isna() ])
 print("Unique KW: {}".format(len(un_KW)))
 print("Out of: {}".format(tot_KW))
print("Samples with no KW: {}".format(len(df[df['keyword'].isna() ])))
 print("Unique LOC: {}".format(len(un_LOC)))
 print("Out of: {}".format(tot_LOC))
print("Samples with no Loc: {}".format(len(df[df['location'].isna() ])))
n_fold = 6
random_state = 802<predict_on_test>
disaster_keywords = [kw for kw in df.loc[df.target == 1].keyword]
regular_keywords = [kw for kw in df.loc[df.target == 0].keyword]
disaster_keywords_counts = dict(pd.DataFrame(data={'x': disaster_keywords} ).x.value_counts())
 regular_keywords_counts = dict(pd.DataFrame(data={'x': regular_keywords} ).x.value_counts())
 all_keywords_counts = dict(pd.DataFrame(data={'x': df.keyword.values} ).x.value_counts())
 for keyword, _ in sorted(all_keywords_counts.items() , key=lambda x: x[1], reverse=True)[:10]:
print("> KW: {}".format(keyword))
print("--
print("--
print('--------')
ttf_est_train, ttf_est_cv, ttf_est_oof, _ = get_prediction(X_bsp_train,
y_bsp_train,
X_test=X_bsp_cv,
n_fold=n_fold,
random_state=random_state)
 print("TTF CV MAE: {:.7f}"\ 
.format(np.abs(y_bsp_cv['time_to_failure'] - ttf_est_cv ).mean()))
print("TTF OOF MAE: {:.7f}"\ 
.format(np.abs(y_bsp_train['time_to_failure'] - ttf_est_oof ).mean()))<predict_on_test>
def clean_df(df):
def remove_stopwords(text):
if text is not None:
tokens = [x for x in word_tokenize(text)if x not in stop]
return " ".join(tokens)
 else:
return None
df["text"] = df['text'].apply(lambda x : x.lower())
 def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 df['text']=df['text'].apply(lambda x : remove_URL(x))
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 df['text']=df['text'].apply(lambda x : remove_html(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df['text']=df['text'].apply(lambda x: remove_emoji(x))
def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 df['text']=df['text'].apply(lambda x : remove_punct(x))
df.text = df.text.replace('\s+', ' ', regex=True)
 return df
df = clean_df(df)
 print("-- Word distrig Positive Class")
 show_word_distrib(target=1, field="text")
 print("-- Word distrib Negative Class")
 show_word_distrib(target=0, field="text")
taf_est_train, taf_est_cv, taf_est_oof, _ = get_prediction(X_bsp_train,
y_bsp_train_new,
X_test=X_bsp_cv,
n_fold=n_fold,
random_state=random_state )<concatenate>
def read_test(test_path="/kaggle/input/nlp-getting-started/test.csv"):
my_df = pd.read_csv(test_path)
 res_df = my_df[['id']]
my_df = my_df[['text']]
add_hashtags(my_df)
 my_df = clean_df(my_df)
 print("Test DF: {}".format(my_df.head(10)))
 return my_df, res_df
def dump_preds(res_df, preds, out="default"):
res_df['target'] = None
for i, p in enumerate(preds):
res_df.ix[i, 'target'] = p
res_df.to_csv(out, index = False)
 def split_data(df, _t=True):
X = df.text
if _t:
Y = df.target
le = LabelEncoder()
Y = le.fit_transform(Y)
 Y = Y.reshape(-1,1)
 return X, Y
else:
return X
X_bsp_train = X_bsp_train.assign(taf_estimate = taf_est_oof);
X_bsp_cv = X_bsp_cv.assign(taf_estimate = taf_est_cv);<compute_train_metric>

 def build_tfidf_logreg(df):
my_df = df[['text','target']]
x_features = my_df.columns[0]
x_data = my_df[x_features]
Y = my_df["target"]
x_train, x_validation, y_train, y_validation = model_selection.train_test_split(
 x_data.values, Y.values, test_size=0.2, random_state=7)
 tfidf_vectorizer = TfidfVectorizer(
 analyzer='word',
tokenizer=lambda x: x,
preprocessor=lambda x: x,
token_pattern=None)
 lr = LogisticRegression()
tfidf_lr_pipe = Pipeline([('tfidf', tfidf_vectorizer),('lr', lr)])
 tfidf_lr_pipe.fit(x_train, y_train)
 return tfidf_lr_pipe
def test_tfidf_logreg(model, test_path="/kaggle/input/nlp-getting-started/test.csv"):
my_df, res_df = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 x_data = my_df["text"].values
preds = model.predict(x_data)
 return res_df

 def test_majority_model(test_path="/kaggle/input/nlp-getting-started/test.csv"):
my_df = pd.read_csv(test_path)
 res = my_df[['id']]
res['target'] = 1
res.to_csv("res_majority.csv", index = False)
 return res
ttf_est_train, ttf_est_cv, ttf_est_oof, model_trial = get_prediction(X_bsp_train, y_bsp_train,
X_test=X_bsp_cv,
n_fold=n_fold,random_state=random_state)
 print("TTF CV MAE after adding TAF to training features: {:.7f}"\ 
.format(np.abs(y_bsp_cv['time_to_failure'] - ttf_est_cv ).mean()))
print("TTF OOF MAE after adding TAF to training features: {:.7f}"\ 
.format(np.abs(y_bsp_train['time_to_failure'] - ttf_est_oof ).mean()))<import_modules>
X_train, y_train = split_data(df)
 test_df, res_df = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 X_test = split_data(test_df, _t=False)
 text_clf = Pipeline([('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('clf', GradientBoostingClassifier(n_estimators=100)) ,
])
from eli5 import show_weights
from eli5.sklearn import PermutationImportance
from IPython.display import display<train_model>
%matplotlib inline
max_words = 750
max_len = 160
def process_data(X, tok=None):
if tok is None:
tok = Tokenizer(num_words=max_words)
 tok.fit_on_texts(X)
 sequences = tok.texts_to_sequences(X)
 sequences_matrix = sequence.pad_sequences(sequences,maxlen=max_len)
 return sequences_matrix, tok
def RNN() :
inputs = Input(name='inputs',shape=[max_len])
 layer = Embedding(max_words,50,input_length=max_len )(inputs)
 layer = LSTM(64 )(layer)
 layer = Dense(256,name='FC1' )(layer)
 layer = Activation('relu' )(layer)
 layer = Dropout(0.5 )(layer)
 layer = Dense(1,name='out_layer' )(layer)
 layer = Activation('sigmoid' )(layer)
 model = Model(inputs=inputs,outputs=layer)
 return model
def test_model(model, tok=None, test_path="/kaggle/input/nlp-getting-started/test.csv", cut=0.5):
my_df, res = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 X = split_data(my_df, _t=False)
 sequences_matrix, tok = process_data(X, tok=tok)
 preds = model.predict(sequences_matrix)
 normalized_preds = []
for p in preds:
if p >= cut:
normalized_preds.append(1)
 else:
normalized_preds.append(0)
 return res
def tune_cutoff(model, tok):
x_tune = df['text'].values
x_target = df[['target']]
X, Y = split_data(df)
 sequences_matrix, tok = process_data(X, tok=tok)
 preds = model.predict(sequences_matrix)
 x_target["preds"] = preds
accumulator = 0
_0 = []
max_0 = 0
_1 = []
min_1 = 0
for i, row in x_target.iterrows() :
if row['target'] == 0:
_0.append(row['preds'])
 if row['preds'] > max_0:
max_0 = row['preds']
else:
_1.append(row['preds'])
 if row['preds'] < min_1:
min_1 = row['preds']
mean_0 = sum(_0)/len(_0)
 mean_1 = sum(_1)/len(_1)
 return max(mean_0, mean_1)/min(mean_0, mean_1)
perm = PermutationImportance(model_trial, random_state=1 ).fit(X_bsp_cv,y_bsp_cv)
 display(show_weights(perm, feature_names = X_bsp_cv.columns.tolist()))<prepare_output>
vocab_size = 10000
embedding_dim = 100
max_length = 50
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
validation_split = 0.10
def load_glove_vectors() :
print('Indexing word vectors.')
 GLOVE_DIR = '/kaggle/input/glove6b100dtxt/'
embeddings_index = {}
print(list(os.walk("/kaggle/input")))
 with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt')) as f:
for line in f:
word, coefs = line.split(maxsplit=1)
 coefs = np.fromstring(coefs, 'f', sep=' ')
 embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
return embeddings_index
def tokenize(df):
tokenizer = Tokenizer(num_words = vocab_size, oov_token = oov_tok)
 tokenizer.fit_on_texts(df.text)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
training_sequences = tokenizer.texts_to_sequences(df.text)
 training_padded = pad_sequences(training_sequences, maxlen = max_length,
padding = padding_type, truncating = trunc_type)
 print('Shape of the data vector is', training_padded.shape, df.target.shape)
 return training_sequences, training_padded, word_index, tokenizer
def prepare_embedding_layer(word_index, embeddings_index):
print('Preparing the embedding matrix')
 num_words = min(vocab_size, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embedding_dim))
for word, index in word_index.items() :
if index >= vocab_size:
continue
embedding_vector = embeddings_index.get(word, np.zeros(embedding_dim, dtype='float32'))
if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
embedding_layer = Embedding(num_words, embedding_dim,
embeddings_initializer = Constant(embedding_matrix),
input_length = max_length,
trainable = False)
 return embedding_layer
def model(embedding_layer):
METRICS = [
metrics.BinaryAccuracy(name='accuracy'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.AUC(name='auc')]
sequence_input = Input(shape =(max_length,))
embedded_sequences = embedding_layer(sequence_input)
 x = Bidirectional(tf.keras.layers.LSTM(64))(embedded_sequences)
 x = Dropout(0.5 )(x)
 x = Dense(32, activation = 'relu' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(1, activation = 'sigmoid' )(x)
 model = Model(sequence_input, output)
 model.compile(loss='binary_crossentropy',optimizer=tf.keras.optimizers.Adam(lr =.0002),metrics = METRICS)
 return model
def test_model(model, test_df, tokenizer):
test_sequences = tokenizer.texts_to_sequences(test_df.text)
 test_padded = pad_sequences(test_sequences, maxlen = max_length,
padding = padding_type, truncating = trunc_type)
 predictions = model.predict(test_padded)
 predictions = np.round(predictions ).astype(int ).flatten()
dump_preds(res, predictions, out="submission.csv")
X_bsp_train['ttf_estimate'] = ttf_est_oof
X_bsp_cv['ttf_estimate'] = ttf_est_cv<find_best_model_class>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]

 kernel_size = 5
filters = 64
pool_size = 4
x = Dropout(0.1 )(sequence_output)
 x = Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1 )(x)
 x = MaxPooling1D(pool_size=pool_size )(x)
 x = Bidirectional(tf.keras.layers.LSTM(64))(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
test_df, res = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encode(df.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test_df.text.values, tokenizer, max_len=160)
 train_labels = df.target.values
model = build_model(bert_layer, max_len=160)
 model.summary()
early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor='val_auc',
verbose=1,
patience=4,
mode='max',
restore_best_weights=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.15,
epochs=20,
batch_size=16,
callbacks=[early_stopping] 
)
 test_pred = model.predict(test_input)
 _t = []
for t in test_pred:
if t < 0.5:
_t.append(0)
 else:
_t.append(1)
 dump_preds(res, _t, out="submission.csv")
num_bootstrap = 2
coeff_bsp = [0.4, 0.6]
for b in tqdm_notebook(range(num_bootstrap)) :
print("
Bootstrapping iteration {0}.".format(b+1))
random_state = b
taf_est_train, taf_est_cv, taf_est_oof, model_taf = \
get_prediction(X_bsp_train.drop(columns=['taf_estimate']),
y_bsp_train_new,
X_test=X_bsp_cv.drop(columns=['taf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TAF training OOF MAE: {:.7f}"\ 
.format(np.abs(y_bsp_train_new['taf'] - taf_est_oof ).mean()))
print("TAF CV MAE: {:.7f}"\ 
.format(np.abs(y_bsp_cv_new['taf'] - taf_est_cv ).mean()))
print("Mean abs change in TAF feature for CV: {:.7f}"\ 
.format(np.abs(X_bsp_cv['taf_estimate'] - taf_est_cv ).mean()))
X_bsp_train['taf_estimate'] = coeff_bsp[0]*X_bsp_train['taf_estimate'] + coeff_bsp[1]*taf_est_oof
X_bsp_cv['taf_estimate'] = coeff_bsp[0]*X_bsp_cv['taf_estimate'] + coeff_bsp[1]*taf_est_cv
ttf_est_train, ttf_est_cv, ttf_est_oof, model = \
get_prediction(X_bsp_train.drop(columns=['ttf_estimate']),
y_bsp_train,
X_test=X_bsp_cv.drop(columns=['ttf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TTF training OOF MAE: {:.7f}"\ 
.format(np.abs(y_bsp_train['time_to_failure'] - ttf_est_oof ).mean()))
print("TTF CV MAE: {:.7f}"\ 
.format(np.abs(y_bsp_cv['time_to_failure'] - ttf_est_cv ).mean()))
print("Mean abs change in TTF feature for CV: {:.7f}"\ 
.format(np.abs(X_bsp_cv['ttf_estimate'] - ttf_est_cv ).mean()))
X_bsp_train['ttf_estimate'] = coeff_bsp[0]*X_bsp_train['ttf_estimate'] + coeff_bsp[1]*ttf_est_oof
X_bsp_cv['ttf_estimate'] = coeff_bsp[0]*X_bsp_cv['ttf_estimate'] + coeff_bsp[1]*ttf_est_cv<features_selection>
!pip install simpletransformers
!git clone --recursive https://github.com/NVIDIA/apex.git
!pip install --upgrade --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" apex/.
 shutil.rmtree('apex')
 if os.path.exists("cache_dir"):
shutil.rmtree('cache_dir')
 if os.path.exists("outputs"):
shutil.rmtree('outputs')
 if os.path.exists("runs"):
shutil.rmtree('runs')
 seed = 98
important_feature_index = np.argsort(model.feature_importances_)[::-1]
X_bsp_train.columns[important_feature_index[:20]]<prepare_x_and_y>
!pip install tamnun
class DenseTransformer(TransformerMixin):
def fit(self, X, y=None, **fit_params):
return self
def transform(self, X, y=None, **fit_params):
return X.todense()
def get_models() :
models = []
tfidf_vectorizer = TfidfVectorizer(
 analyzer='word',
tokenizer=lambda x: x,
preprocessor=lambda x: x,
token_pattern=None 
)
 lr = LogisticRegression()
tfidf_lr_pipe = Pipeline([('tfidf', tfidf_vectorizer),('lr', lr)])
 dec_pipe = Pipeline([('tfidf', tfidf_vectorizer),('dt', DecisionTreeClassifier())])
 svc = Pipeline([('tfidf', tfidf_vectorizer),('svc', SVC(gamma='scale', probability=True)) ])
 gaus = Pipeline([('tfidf', tfidf_vectorizer),('to_dense', DenseTransformer()),('gaus', GaussianNB())])
 kn = Pipeline([('tfidf', tfidf_vectorizer),('kn', KNeighborsClassifier())])
 ada = Pipeline([('tfidf', tfidf_vectorizer),('ada',AdaBoostClassifier())])
 bagging = Pipeline([('tfidf', tfidf_vectorizer),('bag',BaggingClassifier(n_estimators=10)) ])
 ran_forest = Pipeline([('tfidf', tfidf_vectorizer),('ran',RandomForestClassifier(n_estimators=10)) ])
 extra_tree = Pipeline([('tfidf', tfidf_vectorizer),('extr',ExtraTreesClassifier(n_estimators=10)) ])
 gradient_boosting = Pipeline([('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('clf', GradientBoostingClassifier(n_estimators=100)) ,
])
 module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 model = build_model(bert_layer, max_len=160)
 model.summary()
models.append(model)
 models.append(dec_pipe)
 models.append(tfidf_lr_pipe)
 models.append(svc)
 models.append(gaus)
 models.append(kn)
 models.append(ada)
 models.append(bagging)
 models.append(ran_forest)
 models.append(extra_tree)
 models.append(gradient_boosting)
 return models, tokenizer
def get_out_of_fold_predictions(X, y, models, tokenizer):
	meta_X, meta_y = list() , list()
	
	kfold = StratifiedKFold(n_splits=5, shuffle=True)
 	
	for train_ix, test_ix in kfold.split(X, y):
		fold_yhats = list()
		train_X, test_X = X[train_ix], X[test_ix]
		train_y, test_y = y[train_ix], y[test_ix]
		meta_y.extend(test_y)
 		
		for i, model in enumerate(models):
			if i == 0:
				train_input = bert_encode(train_X, tokenizer, max_len=160)
 				early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_auc', verbose=1, patience=10, mode='max', restore_best_weights=True)
 				model.fit(train_input, train_y, validation_split=0.25, epochs=10,batch_size=16, callbacks=[early_stopping])
 				_test_X = bert_encode(test_X, tokenizer, max_len=160)
 				yhat = model.predict(_test_X)
 				_y = []
				for __y in yhat:
					one_prob = __y[0]
					zero_prob = 1 - one_prob
					_y.append([zero_prob, one_prob])
 				yhat = _y
			else:
				model.fit(train_X, train_y)
 				yhat = model.predict_proba(test_X)
 			
			fold_yhats.append(yhat)
 		
		meta_X.append(hstack(fold_yhats))
	return vstack(meta_X), asarray(meta_y)
 def fit_base_models(X, y, models, bert_tok):
	for i,model in enumerate(models):
		if i == 0:
			train_input = bert_encode(X, bert_tok, max_len=160)
 			early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_auc', verbose=1, patience=10, mode='max', restore_best_weights=True)
 			model.fit(train_input, y, validation_split=0.10, epochs=10,batch_size=16, callbacks=[early_stopping])
 		else:
			model.fit(X, y)
 def fit_meta_model(X, y):
	model = ExtraTreesClassifier(n_estimators=30)
 	model.fit(X, y)
 	return model
def super_learner_predictions(X, models, meta_model, bert_tok):
	meta_X = list()
	for i, model in enumerate(models):
		if i == 0:
			x = bert_encode(X, bert_tok, max_len=160)
 			yhat = model.predict(x)
 			_y = []
			for y in yhat:
				one_prob = y[0]
				zero_prob = 1 - one_prob
				_y.append([zero_prob, one_prob])
 			yhat =_y
		else:
			yhat = model.predict_proba(X)
 		meta_X.append(yhat)
 	meta_X = hstack(meta_X)
 	
	return meta_model.predict(meta_X)
X_train = train_features_[features_select]
X_test = test_features_[features_select]<train_model>
!pip install flair
def generateTrainingData(dataset, batch_size, max_length, num_classes, emb_size,
stacked_embedding):
x_batch = []
y_batch = []
while True:
data = dataset.sample(frac=1)
 for index, row in data.iterrows() :
my_sent = row["text"]
sentence = Sentence(my_sent)
 stacked_embedding.embed(sentence)
 x = []
for token in sentence:
x.append(token.embedding.cpu().detach().numpy())
 if len(x)== max_length:
break
while len(x)< max_length:
x.append(np.zeros(emb_size))
y = np.zeros(num_classes)
 y[row["target"]] = 1
x_batch.append(x)
 y_batch.append(y)
 if len(y_batch)== batch_size:
yield np.array(x_batch), np.array(y_batch)
 x_batch = []
y_batch = []
def generatePredictionData(dataset, batch_size, max_length, num_classes, emb_size, stacked_embedding):
x_batch = []
while True:
for text in dataset['text'].values:
my_sent = text
sentence = Sentence(my_sent)
 stacked_embedding.embed(sentence)
 x = []
for token in sentence:
x.append(token.embedding.cpu().detach().numpy())
 if len(x)== max_length:
break
while len(x)< max_length:
x.append(np.zeros(emb_size))
x_batch.append(x)
 if len(x_batch)== batch_size:
yield np.array(x_batch)
 x_batch = []
def get_stacked_embeddings() :
stacked_embedding = StackedEmbeddings([ELMoEmbeddings() , WordEmbeddings("en")])
 print("Stacked embedding size: {}".format(stacked_embedding.embedding_length))
embedding_size = stacked_embedding.embedding_length
return stacked_embedding, embedding_size
def declare_model(batch_size, max_len, emb_size, gru_size, num_classes):
sample = Input(batch_shape=(batch_size, max_len, emb_size))
gru_out = Bidirectional(GRU(gru_size, return_sequences=True))(sample)
 gru_out = Flatten()(gru_out)
 predictions = Dense(num_classes, activation='sigmoid' )(gru_out)
 model = Model(inputs=sample, outputs=[predictions])
 model.compile(optimizer=Adam() ,loss='binary_crossentropy', metrics=["acc"])
 print(model.summary())
 return model