Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<sort_values>
tweets = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
corrs = train.corr()
corrs['Target'].sort_values(ascending = True ).head(10 )<sort_values>
tweets[tweets.sent_len > 8]
tweets[tweets.word_len > 50]
corrs['Target'].sort_values(ascending = True ).dropna().tail(10 )<filter>
stop = set(stopwords.words('english'))
corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(tweet)]
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(tweet)]
def count_top_stopwords(corpus):
stopwords_freq = {}
for word in corpus:
if word in stop:
if word in stopwords_freq:
stopwords_freq[word] += 1
else:
stopwords_freq[word] = 1
topwords = sorted(stopwords_freq.items() , key=lambda item: item[1], reverse=True)[:10]
x,y = zip(*topwords)
 return x,y
x0,y0 = count_top_stopwords(corpus0)
 x1,y1 = count_top_stopwords(corpus1)
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 plt.bar(x0,y0, color=color[0])
 plt.title('Top Stopwords for Non-Disaster Tweets')
 plt.subplot(1,2,2)
 plt.bar(x1,y1, color=color[1])
 plt.title('Top Stopwords for Disaster Tweets')
train = train[train['idhogar'].isin(list(train_valid['idhogar'])) ]
train.head()<filter>
stop = ENGLISH_STOP_WORDS.union(stop)
 def remove_url(txt):
return " ".join(re.sub("([^0-9A-Za-z \t])|(\w+:\/\/\S+)", "", txt ).split())
 corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(remove_url(tweet)) ]
corpus0 = list(filter(lambda x: x not in stop, corpus0))
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(remove_url(tweet)) ]
corpus1 = list(filter(lambda x: x not in stop, corpus1))
a = Counter(corpus0 ).most_common()
df0 = pd.DataFrame(a, columns=['Word','Count'])
 a = Counter(corpus1 ).most_common()
df1 = pd.DataFrame(a, columns=['Word','Count'])
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 sns.barplot(x='Word',y='Count',data=df0.head(10), color=color[1] ).set_title('Most Common Words for Non-Disasters')
 plt.xticks(rotation=45)
 plt.subplot(1,2,2)
 sns.barplot(x='Word',y='Count',data=df1.head(10), color=color[0] ).set_title('Most Common Words for Disasters')
 plt.xticks(rotation=45)
test = test[test['idhogar'].isin(list(test_valid['idhogar'])) ]
test.head()<drop_column>
tweets['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in tweets.text]
tweets['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in tweets.text]
tweets['exclaimation_num'] = [tweet.count('!')for tweet in tweets.text]
tweets['questionmark_num'] = [tweet.count('?')for tweet in tweets.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in tweets.text])
 tweets = tweets.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
tweets['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in tweets.text]
train_labels = np.array(train.pop('Target')).reshape(( -1,))
test_ids = list(test.pop('idhogar'))<concatenate>
tweets.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
tweets.text = [decontraction(tweet)for tweet in tweets.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
tweets.text = tweets.text.apply(lambda x: remove_emoji(x))
train, test = train.align(test, axis = 1, join = 'inner')
 all_features = list(train.columns)
 train.shape<set_options>
tweets.text = tweets.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 tweets.text = tweets.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
tweets.text = tweets.text.apply(lambda x: replace_amp(x))
%%capture 
<compute_test_metric>
tweets = pd.read_csv('.. /input/twitter-logo/tweets_segmented.csv' )
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True<import_modules>
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 tweets.text = tweets.text.apply(lambda x: lemma(x))
from IPython.display import display<choose_model_class>
def generate_ngrams(text, n):
words = word_tokenize(text)
 return [' '.join(ngram)for ngram in list(get_data(ngrams(words, n)))if not all(w in stop for w in ngram)]
def get_data(gen):
try:
for elem in gen:
yield elem
except(RuntimeError, StopIteration):
return
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 hyp_OPTaaS = { 'boosting_type': 'dart',
'colsample_bytree': 0.9843467236959204,
'learning_rate': 0.11598629586769524,
'min_child_samples': 44,
'num_leaves': 49,
'reg_alpha': 0.35397370408131534,
'reg_lambda': 0.5904910774606467,
'subsample': 0.6299872254632797,
'subsample_for_bin': 60611}
model = lgb.LGBMClassifier(**hyp_OPTaaS, class_weight = 'balanced',
objective = 'multiclass', n_jobs = -1, n_estimators = 10000)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores<create_dataframe>
bigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_d_dict = {}
for bgs in bigrams_disaster:
for bg in bgs:
if bg in bigrams_d_dict:
bigrams_d_dict[bg] += 1
else:
bigrams_d_dict[bg] = 1
bigrams_d_df = pd.DataFrame(bigrams_d_dict.items() , columns=['Bigrams','Count'])
 bigrams_nd_dict = {}
for bgs in bigrams_ndisaster:
for bg in bgs:
if bg in bigrams_nd_dict:
bigrams_nd_dict[bg] += 1
else:
bigrams_nd_dict[bg] = 1
bigrams_nd_df = pd.DataFrame(bigrams_nd_dict.items() , columns=['Bigrams','Count'] )
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = pd.DataFrame({'version': ['default_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]} )<save_to_csv>
trigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_d_dict = {}
for tgs in trigrams_disaster:
for tg in tgs:
if tg in trigrams_d_dict:
trigrams_d_dict[tg] += 1
else:
trigrams_d_dict[tg] = 1
trigrams_d_df = pd.DataFrame(trigrams_d_dict.items() , columns=['Trigrams','Count'])
 trigrams_nd_dict = {}
for tgs in trigrams_ndisaster:
for tg in tgs:
if tg in trigrams_nd_dict:
trigrams_nd_dict[tg] += 1
else:
trigrams_nd_dict[tg] = 1
trigrams_nd_df = pd.DataFrame(trigrams_nd_dict.items() , columns=['Trigrams','Count'] )
submission.to_csv('ft_baseline.csv', index = False )<categorify>
def remove_stopwords(text):
word_tokens = word_tokenize(text)
 return ' '.join([w.lower() for w in word_tokens if not w.lower() in stop])
 tweets['text_nostopwords'] = tweets.text.apply(lambda x: remove_stopwords(x))
def range_calc(numeric):
return np.max(numeric)- np.min(numeric)
 range_ = make_agg_primitive(function = range_calc,
input_types = [ft.variable_types.Numeric],
return_type = ft.variable_types.Numeric )<compute_test_metric>
mask = np.array(Image.open('.. /input/twitter-logo/Twitter-Logo_white.png'))
reverse = mask[...,::-1,:]
def wc_words(target, mask=mask):
words = [word.lower() for tweet in tweets[tweets.target == target].text_nostopwords for word in tweet.split() ]
words = list(filter(lambda w: w != 'like', words))
words = list(filter(lambda w: w != 'new', words))
words = list(filter(lambda w: w != 'people', words))
dict = {}
for w in words:
if w in dict:
dict[w] += 1
else:
dict[w] = 1
my_cloud = WordCloud(background_color='white', stopwords=stop, mask=mask, random_state=0 ).generate_from_frequencies(dict)
 plt.subplot(1,2,target+1)
 plt.imshow(my_cloud, interpolation='bilinear')
 plt.axis("off")
 plt.figure(figsize=(15,10))
wc_words(0)
 plt.title('Non-Disaster')
 wc_words(1, reverse)
 plt.title('Disaster')
 plt.show()
def p_corr_calc(numeric1, numeric2):
return np.corrcoef(numeric1, numeric2)[0, 1]
pcorr_ = make_agg_primitive(function = p_corr_calc,
input_types = [ft.variable_types.Numeric, ft.variable_types.Numeric],
return_type = ft.variable_types.Numeric )<compute_test_metric>
pd.reset_option('max_colwidth')
 tweets.drop('text_nostopwords', axis=1, inplace=True)
 tweets.head()
def s_corr_calc(numeric1, numeric2):
return spearmanr(numeric1, numeric2)[0]
scorr_ = make_agg_primitive(function = s_corr_calc,
input_types = [ft.variable_types.Numeric, ft.variable_types.Numeric],
return_type = ft.variable_types.Numeric )<create_dataframe>
X_train, X_val, y_train, y_val = train_test_split(tweets.drop(['id','keyword','location','target'],axis=1), tweets[['target']], test_size=0.2, stratify=tweets[['target']], random_state=0)
 X_train_text = X_train['text']
X_val_text = X_val['text']
print('X_train shape: ', X_train.shape)
 print('X_val shape: ', X_val.shape)
 print('y_train shape: ', y_train.shape)
 print('y_val shape: ', y_val.shape )
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_],
trans_primitives = [], drop_exact = all_features,
max_depth = 2,
verbose = 1, n_jobs = -1,
chunk_size = 100 )<concatenate>
print('Train Class Proportion:
', y_train['target'].value_counts() / len(y_train)* 100)
 print('
Validation Class Proportion:
', y_val['target'].value_counts() / len(y_val)* 100 )
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 feature_matrix.shape<count_duplicates>
tokenizer_1 = Tokenizer(num_words=5000, oov_token='<UNK>')
 tokenizer_1.fit_on_texts(X_train_text )
def post_process(feature_matrix,
missing_threshold = 0.95,
correlation_threshold = 0.95):
start_features = feature_matrix.shape[1]
feature_matrix = feature_matrix.iloc[:, ~feature_matrix.columns.duplicated() ]
n_duplicated = start_features - feature_matrix.shape[1]
print(f'There were {n_duplicated} duplicated features.')
 feature_matrix = feature_matrix.replace({np.inf: np.nan, -np.inf:np.nan} ).reset_index()
ids = list(feature_matrix.pop('idhogar'))
labels = list(feature_matrix.pop('Target'))
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
feature_matrix = pd.get_dummies(feature_matrix)
 n_features_start = feature_matrix.shape[1]
print('Original shape: ', feature_matrix.shape)
 missing = pd.DataFrame(feature_matrix.isnull().sum())
 missing['fraction'] = missing[0] / feature_matrix.shape[0]
missing.sort_values('fraction', ascending = False, inplace = True)
 missing_cols = list(missing[missing['fraction'] > missing_threshold].index)
 n_missing_cols = len(missing_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in missing_cols]]
print('{} missing columns with threshold: {}.'.format(n_missing_cols, missing_threshold))
unique_counts = pd.DataFrame(feature_matrix.nunique() ).sort_values(0, ascending = True)
 zero_variance_cols = list(unique_counts[unique_counts[0] == 1].index)
 n_zero_variance_cols = len(zero_variance_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in zero_variance_cols]]
print('{} zero variance columns.'.format(n_zero_variance_cols))
corr_matrix = feature_matrix.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
n_collinear = len(to_drop)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in to_drop]]
print('{} collinear columns removed with correlation above {}.'.format(n_collinear, correlation_threshold))
total_removed = n_duplicated, n_missing_cols + n_zero_variance_cols + n_collinear
print('Total columns removed: ', total_removed)
 print('Shape after feature selection: {}.'.format(feature_matrix.shape))
feature_matrix['idhogar'] = ids
feature_matrix['Target'] = labels
train = feature_matrix[feature_matrix['Target'].notnull() ]
test = feature_matrix[feature_matrix['Target'].isnull() ]
train = train[train['idhogar'].isin(list(train_valid['idhogar'])) ]
test = test[test['idhogar'].isin(list(test_valid['idhogar'])) ]
train_labels = np.array(train.pop('Target')).reshape(( -1,))
test_ids = list(test.pop('idhogar'))
train, test = train.align(test, join = 'inner', axis = 1)
 assert(len(train_labels)== train.shape[0]), "Labels must be same length as number of training observations"
assert(len(test_ids)== test.shape[0]), "Must be equal number of test ids as testing observations"
return train, train_labels, test, test_ids<define_variables>
X_train_text = tokenizer_1.texts_to_sequences(X_train_text)
 X_val_text = tokenizer_1.texts_to_sequences(X_val_text)
 print(X_train_text[:10])
 print('')
 print(X_val_text[:10] )
train, train_labels, test, test_ids = post_process(feature_matrix )<create_dataframe>
tokenizer_1.sequences_to_texts([X_train_text[1]] )
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = [], drop_contains = list(all_features),
max_depth = 2, max_features = 1000,
verbose = 1, n_jobs = -1,
chunk_size = 100 )<concatenate>
vocab_size = len(tokenizer_1.word_index)+ 1
embeddings_index = dict()
f = open('.. /input/glove-global-vectors-for-word-representation/glove.twitter.27B.200d.txt')
 for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape<create_dataframe>
embedding_matrix = np.zeros(( vocab_size, 200))
for word, i in tokenizer_1.word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Embedding Matrix Shape:', embedding_matrix.shape )
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['additionalft_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))<create_dataframe>
num_epochs=15
dropout=0.2
recurrent_dropout=0.2
lr=0.0005
batch_size=128
class_weight = {0: y_train['target'].value_counts() [1]/len(y_train), 1: y_train['target'].value_counts() [0]/len(y_train)}
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_contains = list(all_features),
max_depth = 2, max_features = 1000,
verbose = 1, n_jobs = -1,
chunk_size = 1000 )<concatenate>
lstm_model = Sequential()
embedding_layer = Embedding(vocab_size, 200, weights=[embedding_matrix], input_length=maxlen, trainable=False)
 lstm_model.add(embedding_layer)
 lstm_model.add(LSTM(128, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout))
lstm_model.add(LSTM(128))
lstm_model.add(Dense(1, activation='sigmoid'))
adam = optimizers.Adam(lr=lr)
 lstm_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(lstm_model.summary())
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape<save_to_csv>
checkpoint = ModelCheckpoint('lstm_model.h5', monitor='val_acc', save_best_only=True)
 history = lstm_model.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history )
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide1000_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide1000_featuretools.csv', index = False )<create_dataframe>
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_contains = list(all_features),
max_depth = 2, max_features = 1500,
verbose = 1, n_jobs = -1,
chunk_size = 1000 )<concatenate>
class attention(Layer):
def __init__(self,**kwargs):
super(attention,self ).__init__(**kwargs)
 def build(self,input_shape):
self.W=self.add_weight(name="att_weight",shape=(input_shape[-1],1),initializer="normal")
 self.b=self.add_weight(name="att_bias",shape=(input_shape[1],1),initializer="zeros")
 super(attention, self ).build(input_shape)
 def call(self,x):
et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)
 at=K.softmax(et)
 at=K.expand_dims(at,axis=-1)
 output=x*at
return K.sum(output,axis=1)
 def compute_output_shape(self,input_shape):
return(input_shape[0],input_shape[-1])
 def get_config(self):
return super(attention,self ).get_config()
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape<save_to_csv>
num_epochs=15
dropout=0.3
recurrent_dropout=0.3
lr=0.0005
batch_size=128
sequence_input = Input(shape=(maxlen,))
embedded_sequences = Embedding(vocab_size, 200, weights=[embedding_matrix], trainable=False )(sequence_input)
 lstm = Bidirectional(LSTM(128, return_sequences = True, dropout=dropout, recurrent_dropout=recurrent_dropout))(embedded_sequences)
(lstm, forward_h, forward_c, backward_h, backward_c)= Bidirectional(LSTM(128, return_sequences=True, return_state=True))(lstm)
 att_out=attention()(lstm)
 outputs=Dense(1,activation='sigmoid' )(att_out)
 model_attn = Model(sequence_input, outputs)
 adam = optimizers.Adam(lr=lr)
 model_attn.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(model_attn.summary() )
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide1500_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide1500_featuretools.csv', index = False )<create_dataframe>
checkpoint = ModelCheckpoint('attn_model.h5', monitor='val_acc', save_best_only=True)
 history_attn = model_attn.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history_attn )
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_exact = list(all_features),
max_depth = 2, max_features = 2000,
verbose = 1, n_jobs = -1,
chunk_size = 100 )<concatenate>
maxlen = 160
lr = 1e-5
num_epochs = 3
batch_size=16
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape<save_to_csv>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide2000_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide2000_featuretools.csv', index = False )<save_to_csv>
def build_model(bert_layer, max_len=512, lr=1e-5):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=lr), loss='binary_crossentropy', metrics=['accuracy'])
 return model
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 10)
 results = results.append(pd.DataFrame({'version': ['divide2000_10fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide2000_10fold_featuretools.csv', index = False )<create_dataframe>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_contains = list(all_features),
max_depth = 2, max_features = 5000,
verbose = 1, n_jobs = -1,
chunk_size = 100 )<concatenate>
train_input = bert_encode(X_train.text.values, tokenizer, max_len=maxlen)
 val_input = bert_encode(X_val.text.values, tokenizer, max_len=maxlen)
 train_labels = y_train.target.values
val_labels = y_val.target.values
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape<save_to_csv>
bert_model = build_model(bert_layer, max_len=maxlen, lr=lr)
 bert_model.summary()
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide5000_5fold'], 'F1-mean': [valid_scores.mean() ], 'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide5000_featuretools.csv', index = False )<save_to_csv>
checkpoint = ModelCheckpoint('bertmodel.h5', monitor='val_accuracy', save_best_only=True)
 bert_history = bert_model.fit(
 train_input, train_labels,
validation_data=(val_input, val_labels),
epochs=num_epochs,
callbacks=[checkpoint],
batch_size=batch_size
)
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 10)
 results = results.append(pd.DataFrame({'version': ['divide5000_10fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide5000_10fold_featuretools.csv', index = False )<count_duplicates>
clf = RandomForestClassifier(n_estimators=500, max_depth=15, min_samples_split=20, min_samples_leaf=2, n_jobs=-1, random_state=0)
 clf.fit(X_train.drop('text',axis=1), y_train.target.values)
 clf_pred = clf.predict_proba(X_val.drop('text',axis=1))
print('Validation Accuracy:', accuracy_score(y_val.target.values, clf_pred.argmax(axis=-1)) )
feature_matrix = feature_matrix.iloc[:, ~feature_matrix.columns.duplicated() ].reset_index()<filter>
bert_model.load_weights('bertmodel.h5')
 val = X_val.copy()
val = val[['text']]
val['target'] = y_val
val['pred'] = bert_model.predict(val_input)
 val['pred'] = val['pred'].apply(lambda x: 1 if x >=0.5 else 0)
 error = val[val['target'] != val['pred']]
error.head()
train_ids = list(feature_matrix[(feature_matrix['Target'].notnull())&(feature_matrix['idhogar'].isin(list(train_valid['idhogar'])))]['idhogar'] )<save_to_csv>
test['char_len'] = test.text.str.len()
word_tokens = [len(word_tokenize(tweet)) for tweet in test.text]
test['word_len'] = word_tokens
sent_tokens = [len(sent_tokenize(tweet)) for tweet in test.text]
test['sent_len'] = sent_tokens
train['Target'] = train_labels
test['Target'] = np.nan
train['idhogar'] = train_ids
test['idhogar'] = test_ids
data = train.append(test)
 results.to_csv('model_results.csv', index = True)
 data.to_csv('ft_5000_important.csv', index = False )<train_model>
test['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in test.text]
test['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in test.text]
test['exclaimation_num'] = [tweet.count('!')for tweet in test.text]
test['questionmark_num'] = [tweet.count('?')for tweet in test.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in test.text])
 test = test.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
test['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in test.text]
print('Final shape of data(with testing joined to training): ', data.shape )<import_modules>
test.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
test.text = [decontraction(tweet)for tweet in test.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
test.text = test.text.apply(lambda x: remove_emoji(x))
from keras.models import Sequential, Model, load_model
from keras.layers import UpSampling2D, Conv2D, Activation, BatchNormalization, Reshape, Dense, Input, LeakyReLU, Dropout, Flatten, ZeroPadding2D
from keras.optimizers import Adam
import glob
from PIL import Image
import numpy as np
import os
from ast import literal_eval
from scipy.misc import imsave
import xml.etree.ElementTree as ET
from tqdm import tqdm
import zipfile
import time<define_variables>
test.text = test.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 test.text = test.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
test.text = test.text.apply(lambda x: replace_amp(x))
class DataLoader:
def __init__(self, root='.. /input/', image_size=(64, 64)) :
self.root = root
self.breeds = os.listdir(self.root + 'annotation/Annotation/')
 self.idxIn = 0
self.namesIn = []
self.image_size = image_size
self.imagesIn = np.zeros(( 25000, image_size[0], image_size[1], 3))
def load_images(self):
start = time.time()
for breed in self.breeds:
for dog in os.listdir(self.root + 'annotation/Annotation/' + breed):
try:
img = Image.open(self.root + 'all-dogs/all-dogs/' + dog + '.jpg')
 except:
continue
tree = ET.parse(self.root + 'annotation/Annotation/' + breed + '/' + dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( self.image_size[0], self.image_size[1]), Image.ANTIALIAS)
 self.imagesIn[self.idxIn,:,:,:] = np.asarray(img2)
 if self.idxIn % 1000 == 0:
print(self.idxIn)
 self.namesIn.append(breed)
 self.idxIn += 1
idx = np.arange(self.idxIn)
 np.random.shuffle(idx)
 self.imagesIn = self.imagesIn[idx,:,:,:]
self.namesIn = np.array(self.namesIn)[idx]
end = time.time()
print(end - start)
 return self.imagesIn<load_pretrained>
test = pd.read_csv('.. /input/twitter-logo/tweets_test_segmented.csv' )
class ImageZipper:
def __init__(self, folder='./', name='images.zip'):
self.folder = folder
self.z = zipfile.PyZipFile(self.folder + name, mode='w')
 def add_image(self, image, image_name):
file_name = self.folder + image_name + '.png'
imsave(file_name, image)
 self.z.write(file_name)
 os.remove(file_name)
 def close(self):
self.z.close()<init_hyperparams>
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 test.text = test.text.apply(lambda x: lemma(x))
class DCGAN:
def __init__(self, discriminator_path, generator_path, output_directory, data_loader):
self.data_loader = data_loader
self.img_size = self.data_loader.image_size
self.upsample_layers = 5
self.starting_filters = 64
self.kernel_size = 3
self.channels = 3
self.discriminator_path = discriminator_path
self.generator_path = generator_path
self.output_directory = output_directory
self.generated_folder = f"{self.output_directory}/generated_{self.img_size[0]}x{self.img_size[1]}"
def build_generator(self):
noise_shape =(100,)
 model = Sequential()
model.add(
 Dense(self.starting_filters *(self.img_size[0] //(2 ** self.upsample_layers)) *(self.img_size[1] //(2 ** self.upsample_layers)) ,
activation="relu", input_shape=noise_shape))
model.add(Reshape(((self.img_size[0] //(2 ** self.upsample_layers)) , 
(self.img_size[1] //(2 ** self.upsample_layers)) ,
self.starting_filters)))
 model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(1024, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(512, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(256, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(64, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(32, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(self.channels, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("tanh"))
noise = Input(shape=noise_shape)
 img = model(noise)
 return Model(noise, img)
 def build_discriminator(self):
img_shape =(self.img_size[0], self.img_size[1], self.channels)
 model = Sequential()
model.add(Conv2D(32, kernel_size=self.kernel_size, strides=2, input_shape=img_shape, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(64, kernel_size=self.kernel_size, strides=2, padding="same"))
model.add(ZeroPadding2D(padding=(( 0, 1),(0, 1))))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(128, kernel_size=self.kernel_size, strides=2, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(256, kernel_size=self.kernel_size, strides=1, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(512, kernel_size=self.kernel_size, strides=1, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
img = Input(shape=img_shape)
 validity = model(img)
 return Model(img, validity)
 def build_gan(self):
optimizer = Adam(0.0002, 0.5)
 if self.discriminator_path is not None and self.generator_path is not None and os.path.exists(self.discriminator_path)and os.path.exists(self.generator_path):
self.discriminator = load_model(self.discriminator_path)
 self.generator = load_model(self.generator_path)
 print("Loaded models...")
 else:
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 self.generator = self.build_generator()
self.generator.compile(loss='binary_crossentropy', optimizer=optimizer)
 z = Input(shape=(100,))
img = self.generator(z)
 self.discriminator.trainable = False
valid = self.discriminator(img)
 self.combined = Model(z, valid)
 self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
 def train(self, epochs, batch_size=32, save_interval=50):
self.build_gan()
X_train = self.data_loader.load_images()
print("Training Data Shape: ", X_train.shape)
 X_train =(X_train.astype(np.float32)- 127.5)/ 127.5
half_batch = batch_size // 2
for epoch in range(epochs):
noise = np.random.normal(0, 1,(batch_size, 100))
g_loss = self.combined.train_on_batch(noise, np.ones(( batch_size, 1)))
 idx = np.random.randint(0, X_train.shape[0], half_batch)
 imgs = X_train[idx]
noise = np.random.normal(0, 1,(half_batch, 100))
gen_imgs = self.generator.predict(noise)
 d_loss_real = self.discriminator.train_on_batch(imgs, np.ones(( half_batch, 1)))
 d_loss_fake = self.discriminator.train_on_batch(gen_imgs, np.zeros(( half_batch, 1)))
 d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
 if epoch %(save_interval)== 0:
print(f"{epoch} [D loss: {d_loss[0]} | D Accuracy: {100 * d_loss[1]}] [G loss: {g_loss}]")
 def gene_imgs(self, count):
noise = np.random.normal(0, 1,(count, 100))
return self.generator.predict(noise)
 def generate_imgs(self, count, threshold):
imgs = []
for i in tqdm(range(count), desc='Generating images'):
score = [0]
while not(threshold[0] < score[0] < threshold[1]):
img = self.gene_imgs(1)
 score = self.discriminator.predict(img)
 imgs.append(img)
 imgs = np.asarray(imgs ).squeeze()
imgs = 0.5 * imgs + 0.5
return imgs<train_model>
test_text = test['text']
test_text = tokenizer_1.texts_to_sequences(test_text)
 test_text = pad_sequences(test_text, padding='post', maxlen=50)
 print('X_test shape:', test_text.shape )
epochs = 530000
batch_size = 24
save_interval = 1000
output_directory = './'
data_loader = DataLoader()
dcgan = DCGAN(None, None, output_directory, data_loader)
 start = time.time()
dcgan.train(epochs=epochs, batch_size=batch_size, save_interval=save_interval)
 end = time.time()
print(end - start )<choose_model_class>
lstm_model.load_weights('lstm_model.h5')
 submission = test.copy() [['id']]
submission['target'] = lstm_model.predict_classes(test_text)
 submission.to_csv('submission.csv', index=False)
 display(submission.head())
load_generator = "./generat.h5"
load_discriminator = "./discrim.h5"
sample = 10000
sample_thresholds =(0.0, 1.0)
 dcgan = DCGAN(load_discriminator, load_generator, output_directory, data_loader)
 dcgan.build_gan()
zipper = ImageZipper(folder=output_directory)
 for i in range(0, sample, 100):
print('Images generated', i)
 images = dcgan.generate_imgs(100, sample_thresholds)
 for index, image in enumerate(images):
zipper.add_image(image, str(i + index))
zipper.close()<define_variables>
model_attn.load_weights('attn_model.h5')
 submission_attn = test.copy() [['id']]
submission_attn['target'] = model_attn.predict(test_text)
 submission_attn['target'] = submission_attn['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_attn.to_csv('submission_attn.csv', index=False)
 display(submission_attn.head() )
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
else:
x = np.random.choice(np.arange(20000),10000)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<import_modules>
test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 bert_model.load_weights('bertmodel.h5')
 submission_bert = test.copy() [['id']]
submission_bert['target'] = bert_model.predict(test_input)
 submission_bert['target'] = submission_bert['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_bert.to_csv('submission_bert.csv', index=False)
 display(submission_bert.head() )
<categorify><EOS> 
clf_testpred = clf.predict_proba(test.drop(['id','keyword','location','text'],axis=1))
submission_bert = test.copy() [['id']]
submission_bert['target'] =(bert_model.predict(test_input)*0.8 ).ravel() +(clf_testpred.max(axis=1)*0.2)
 submission_bert['target'] = submission_bert['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_bert.to_csv('submission_bert_ensemble.csv', index=False)
 display(submission_bert.head() )