Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<create_dataframe>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
df_train = pd.DataFrame()
df_test = pd.DataFrame()
aggr_mean_list = ['rez_esc', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7', 'parentesco2',
'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',]
other_list = ['escolari', 'age', 'escolari_age']
for item in aggr_mean_list:
group_train_mean = train_set[item].groupby(train_set['idhogar'] ).mean()
group_test_mean = test_set[item].groupby(test_set['idhogar'] ).mean()
new_col = item + '_aggr_mean'
df_train[new_col] = group_train_mean
df_test[new_col] = group_test_mean
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_train = train_set[item].groupby(train_set['idhogar'] ).agg(function)
 group_test = test_set[item].groupby(test_set['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_train[new_col] = group_train
df_test[new_col] = group_test
print(f'new aggregate train set has {df_train.shape[0]} rows, and {df_train.shape[1]} features')
 print(f'new aggregate test set has {df_test.shape[0]} rows, and {df_test.shape[1]} features' )<merge>
!pip3 -q install contractions
!pip3 -q install nb_black
!pip -q install beautifulsoup4
df_test = df_test.reset_index()
df_train = df_train.reset_index()
train_agg = pd.merge(train_set, df_train, on='idhogar')
 test = pd.merge(test_set, df_test, on='idhogar')
 train_agg.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True)
 print(f'new train set has {train_agg.shape[0]} rows, and {train_agg.shape[1]} features')
 print(f'new test set has {test.shape[0]} rows, and {test.shape[1]} features' )<filter>
warnings.filterwarnings("ignore", category=FutureWarning)
train = train_agg.query('parentesco1==1' )<sort_values>
nltk.download("wordnet" )
submission = test[['Id']]
train.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 test.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 correlation = train.corr()
correlation = correlation['Target'].sort_values(ascending=False)
 print(f'The most 20 positive feature:
{correlation.head(20)}')
 print('*'*50)
 print(f'The most 20 negative feature:
{correlation.tail(20)}' )<prepare_x_and_y>
%config IPCompleter.greedy=True
%matplotlib inline
%reload_ext nb_black
y = train['Target']
train.drop(columns=['Target'], inplace=True)
 clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 95, num_leaves = 14, subsample = 0.96 )<train_model>
curr_dir = ".. /input"
already_trained = False
train_file = os.path.join(curr_dir, "nlp-getting-started", "train.csv")
 test_file = os.path.join(curr_dir, "nlp-getting-started", "test.csv")
 sample_submission_file = os.path.join(
 curr_dir, "nlp-getting-started", "sample_submission.csv"
)
kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 predicts_result = []
for train_index, test_index in kf.split(train, y):
print("
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=400, verbose=100)
 predicts_result.append(clf.predict(test))<save_to_csv>
params = {
"legend.fontsize": "x-large",
"figure.figsize":(16, 5),
"axes.labelsize": "x-large",
"axes.titlesize": "x-large",
"xtick.labelsize": "x-large",
"ytick.labelsize": "x-large",
}
pylab.rcParams.update(params )
submission['Target'] = np.array(predicts_result ).mean(axis=0 ).round().astype(int)
 submission.to_csv('submission.csv', index = False )<import_modules>
df = pd.read_csv(train_file)
 df.head()
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold<load_from_csv>
df.isnull().sum()
df = pd.read_csv(".. /input/train.csv" )<categorify>
text_data_duplicate_map = df.text.duplicated().value_counts()
print(text_data_duplicate_map)
 plotBarGraph(
 text_data_duplicate_map,
"Duplicate",
"Frequency",
"Text Data Duplicate or Not",
switch_axis=True,
)
def preprocess(df):

 def mk_categoricals(df, prefixes=None, subsets=None):

 def mk_category(dummies):
assert(dummies.sum(axis=1)<= 1 ).all()
nans = dummies.sum(axis=1)!= 1
if nans.any() :
dummies = dummies.assign(_na=nans.astype(int))
return dummies.idxmax(axis=1 ).astype('category')
 categoricals = pd.DataFrame()
if prefixes:
for prefix in prefixes:
columns = df.columns[df.columns.str.startswith(prefix)]
categoricals[prefix] = mk_category(df[columns])
 if subsets:
for feature_name, subset in subsets.items() :
categoricals[feature_name] = mk_category(df[subset])
 return categoricals
groupper = df.groupby('idhogar')
 interactions =(pd.DataFrame(dict(
 head_escolari=df.parentesco1 * df.escolari,
head_female=df.parentesco1 * df.female,
head_partner_escolari=df.parentesco2 * df.escolari)) 
.groupby(df.idhogar)
.max())
 my_features =(groupper.mean() [['escolari', 'age', 'hogar_nin',
'hogar_total', 'epared3', 'epared1',
'etecho3', 'etecho1', 'eviv3', 'eviv1',
'male',
'r4h1', 'r4h2', 'r4h3', 'r4m1', 'r4m2',
'r4m3',
'r4t1', 'r4t2', 'r4t3', 'v2a1', 'rooms',
'bedrooms',
'meaneduc',
'SQBdependency', 'rez_esc', 'refrig',
'tamviv', 'overcrowding']] 
.join(groupper.std() [['escolari', 'age']],
rsuffix='_std')
.join(groupper[['escolari', 'age']].min() , rsuffix="_min")
.join(groupper[['escolari', 'age']].max() , rsuffix="_max")
.join(groupper[['dis']].sum() , rsuffix="_sum")
.assign(child_rate=lambda x: x.hogar_nin / x.hogar_total,
wrf=lambda x: x.epared3 - x.epared1 +
x.etecho3 - x.etecho1 +
x.eviv3 - x.eviv1,
escolari_range=lambda x: x.escolari_max - x.escolari_min,
age_range=lambda x: x.age_max - x.age_min,
rent_per_individual=lambda x: x.v2a1 / x.r4t3,
rent_per_child=lambda x: x.v2a1 / x.r4t1,
rent_per_over65=lambda x: x.v2a1 / x.r4t3,
rent_per_room=lambda x: x.v2a1 / x.rooms,
rent_per_bedroom=lambda x: x.v2a1 / x.bedrooms,
rooms_per_individual=lambda x: x.rooms / x.r4t3,
rooms_per_child=lambda x: x.rooms / x.r4t1,
bedrooms_per_individual=lambda x: x.bedrooms / x.r4t3,
bedrooms_per_child=lambda x: x.bedrooms / x.r4t1,
years_schooling_per_individual=lambda x: x.escolari / x.r4t3,
years_schooling_per_adult=lambda x: x.escolari /(x.r4t3 - x.r4t1),
years_schooling_per_child=lambda x: x.escolari / x.r4t3 
)
.drop(['hogar_nin', 'hogar_total', 'epared3', 'epared1',
'etecho3', 'etecho1', 'eviv3', 'eviv1'],
axis=1)
.join(interactions)
.join(groupper[['computer', 'television',
'qmobilephone', 'v18q1']] 
.mean().sum(axis=1 ).rename('technics')) 
.assign(technics_per_individual=lambda x: x.technics / x.r4t3,
technics_per_child=lambda x: x.technics / x.r4t1)
.join(mk_categoricals(groupper.mean() ,
prefixes=['lugar', 'area', 'tipovivi',
'energcocinar',
'sanitario', 'pared', 'piso',
'abastagua'],
subsets={'electricity': ['public',
'planpri',
'noelec',
'coopele']})) 
)
 return my_features<data_type_conversions>
print("Original dataframe shape: ", df.shape)
 df.drop_duplicates(subset="text", keep="first", inplace=True)
 print("Dropping duplicate rows and keeping original value shape: ", df.shape)
 df.text.duplicated().value_counts()
X = preprocess(df)
 y = df.groupby('idhogar' ).Target.mean().astype(int)
<load_from_csv>
def search_text_data(query, column="text"):
return df[df[column].str.contains(query)][column]
print(search_text_data("volcano")[:5] )
df_test = pd.read_csv(".. /input/test.csv" ).set_index('Id')
 X_test = preprocess(df_test )<find_best_model_class>
def lower_case_data(data=""):
data = data.lower()
return data
kf = StratifiedKFold(n_splits=5, shuffle=True)
 predicts = []
for train_index, test_index in kf.split(X, y):
print("
X_train, X_val = X.iloc[train_index], X.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=20)
 predicts.append(clf.predict(X_test))<create_dataframe>
def sentences_with_emojis(id_texts):
sentences = []
indeces = id_texts[0]
texts = id_texts[1]
for index, sentence in zip(indeces, texts):
has_emoji = bool(emoji.get_emoji_regexp().search(sentence))
if has_emoji:
sentences.append(( index, sentence))
if len(sentences)== 0:
return "Sentences are clean and don't have emojis!"
else:
return sentences
def clean_emojis(text):
allchars = [str for str in text]
emoji_list = [c for c in allchars if c in emoji.UNICODE_EMOJI]
clean_text = " ".join(
 [str for str in text.split() if not any(i in str for i in emoji_list)] 
)
 return clean_text
predict_by_hh = pd.DataFrame(np.array(predicts ).mean(axis=0 ).round().astype(int),
columns=['Target'],
index=X_test.index )<data_type_conversions>
def clean_urls(text):
text = re.sub(r"https?://\S+", "", text)
 return text
predict = df_test.join(predict_by_hh, on='idhogar')[['Target']].astype(int )<save_to_csv>
def removeSpecialChar(text):
sentence = []
for s in text:
if s == " ":
sentence.append(s)
 if s.isalnum() :
sentence.append(s)
 return "".join(sentence )
predict.to_csv("output.csv" )<set_options>
def checkHtml(text):
return bool(BeautifulSoup(text, "html.parser" ).find())
 html_sentence_map = df["text"].apply(checkHtml ).tolist()
if not any(html_sentence_map):
print("No text containing html found!")
 else:
print("There is some html text!" )
warnings.filterwarnings('ignore' )<load_from_csv>
def remove_accented_chars(text):
new_text =(
 unicodedata.normalize("NFKD", text)
.encode("ascii", "ignore")
.decode("utf-8", "ignore")
)
 return new_text
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test['Target'] = np.nan
data = train.append(test, sort = True )<define_variables>
def remove_words_having_not(words=[""]):
for word in words:
if "not" in word:
words.remove(word)
 return words
stopwords_list = stopwords.words("english")
 print(stopwords_list)
 stopwords_list = remove_words_having_not(
 [removeSpecialChar(contractions.fix(word)) for word in stopwords_list] 
)
 print("
---------------Stopwords after preprocessing---------------
")
 print(stopwords_list )
train_valid = train.loc[train['parentesco1'] == 1, ['idhogar', 'Id', 'Target']].copy()
test_valid = test.loc[test['parentesco1'] == 1, ['idhogar', 'Id']].copy()
submission_base = test[['Id', 'idhogar']]<data_type_conversions>
def remove_stopwords(text):
new_sentence = ""
for word in text.split() :
if word not in stopwords_list:
new_sentence += word + " "
return new_sentence
mapping = {"yes": 1, "no": 0}
data['dependency'] = data['dependency'].replace(mapping ).astype(np.float64)
 data['edjefa'] = data['edjefa'].replace(mapping ).astype(np.float64)
 data['edjefe'] = data['edjefe'].replace(mapping ).astype(np.float64)
 data[['dependency', 'edjefa', 'edjefe']].describe()<feature_engineering>
def remove_numbers(text=""):
new_sentence = ""
for word in text.split() :
num_free_word = "".join([i for i in word if not i.isdigit() ])
 new_sentence += num_free_word + " "
return new_sentence
data['v18q1'] = data['v18q1'].fillna(0)
 data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5<feature_engineering>
def lemmatize(text):
lemmatizer = WordNetLemmatizer()
new_sentence = ""
for word in text.split() :
lematized_word = lemmatizer.lemmatize(word)
 new_sentence += lematized_word + " "
return new_sentence
data['hhsize-diff'] = data['tamviv'] - data['hhsize']
elec = []
for i, row in data.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 data['elec'] = elec
data['elec-missing'] = data['elec'].isnull()
data['walls'] = np.argmax(np.array(data[['epared1', 'epared2', 'epared3']]),
axis = 1)
 data['roof'] = np.argmax(np.array(data[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 data['floor'] = np.argmax(np.array(data[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 data['walls+roof+floor'] = data['walls'] + data['roof'] + data['floor']
data['warning'] = 1 *(data['sanitario1'] + 
(data['elec'] == 0)+
data['pisonotiene'] +
data['abastaguano'] + 
(data['cielorazo'] == 0))
data['bonus'] = 1 *(data['refrig'] +
data['computer'] + 
(data['v18q1'] > 0)+
data['television'])
 data['phones-per-capita'] = data['qmobilephone'] / data['tamviv']
data['tablets-per-capita'] = data['v18q1'] / data['tamviv']
data['rooms-per-capita'] = data['rooms'] / data['tamviv']
data['rent-per-capita'] = data['v2a1'] / data['tamviv']
data['inst'] = np.argmax(np.array(data[[c for c in data if c.startswith('instl')]]), axis = 1)
 data['escolari/age'] = data['escolari'] / data['age']
data['inst/age'] = data['inst'] / data['age']
data['tech'] = data['v18q'] + data['mobilephone']
print('Data shape: ', data.shape )<drop_column>
def remove_white_space(text):
return " ".join(text.split() )
data = data[[x for x in data if not x.startswith('SQB')]]
data = data.drop(columns = ['agesq'])
 data.shape<drop_column>
def clean_sentence_pipeline(text):
text = lower_case_data(text)
 text = clean_emojis(text)
 text = clean_urls(text)
 text = contractions.fix(text)
 text = removeSpecialChar(text)
 text = remove_accented_chars(text)
 text = remove_stopwords(text)
 text = remove_numbers(text)
 text = lemmatize(text)
 return remove_white_space(text)
 def clean_text_column(col_name, df=df):
df["clean_text"] = ""
df["clean_text"] = df[col_name].apply(clean_sentence_pipeline)
 clean_text_column("text" )
data = data.drop(columns = to_drop )<import_modules>
bert_layer = hub.KerasLayer(
 "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1", trainable=True 
)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
import featuretools.variable_types as vtypes<define_variables>
normal_sentence = "covid-19 sucks like hell!"
tokenize_sentence = tokenizer.tokenize(normal_sentence)
 print("Tokens for BERT: ", tokenize_sentence)
 bert_input_sentence = ["[CLS]"] + tokenize_sentence + ["[SEP]"]
print("Token Ids for BERT: ", tokenizer.convert_tokens_to_ids(bert_input_sentence))
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing', 'elec-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin','hhsize-diff',
'elec', 'walls', 'roof', 'floor', 'walls+roof+floor', 'warning', 'bonus',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding',
'phones-per-capita', 'tablets-per-capita', 'rooms-per-capita', 'rent-per-capita']<define_variables>
def bert_encode(texts, max_length=128):
all_token_ids = []
all_input_ids = []
all_segment_ids = []
for sentence in texts:
tokenize_sentence = tokenizer.tokenize(sentence)[
:max_length
]
bert_input_sentence = ["[CLS]"] + tokenize_sentence + ["[SEP]"]
unused_sentence_length = max_length - len(
 bert_input_sentence 
)
 unused_length_array = [0] * unused_sentence_length
token_ids =(
 tokenizer.convert_tokens_to_ids(bert_input_sentence)+ unused_length_array 
)
 bert_input_ids =([1] * len(bert_input_sentence)) + unused_length_array
bert_segment_ids = [0] * max_length
all_token_ids.append(token_ids)
 all_input_ids.append(bert_input_ids)
 all_segment_ids.append(bert_segment_ids)
 return(
 np.array(all_token_ids),
np.array(all_input_ids),
np.array(all_segment_ids),
)
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['age', 'escolari', 'rez_esc', 'inst', 'tech']
ind_cont = ['escolari/age', 'inst/age']<drop_column>
MAX_SEQUENCE_LENGTH = 128
def build_model(hp):
input_word_ids = keras.Input(
 shape=(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name="input_word_ids" 
)
 input_mask = keras.Input(
 shape=(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name="input_mask" 
)
 segment_ids = keras.Input(
 shape=(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name="segment_ids" 
)
 pooled_output, sequence_output = bert_layer(
 [input_word_ids, input_mask, segment_ids] 
)
 clf_output = sequence_output[:, 0, :]
neural_inputs = keras.layers.Dense(
 units=hp.Int(
 "units_hidden_1", min_value=32, max_value=256, step=32, default=128 
),
activation="elu",
kernel_initializer="he_normal",
)(clf_output)
 neural_inputs = keras.layers.Dropout(
 rate=hp.Float("dropout_1", min_value=0.1, max_value=0.9, default=0.2, step=0.1)
 )(neural_inputs)
 neural_inputs = keras.layers.Dense(
 units=hp.Int(
 "units_hidden_2", min_value=32, max_value=256, step=32, default=64 
),
activation="elu",
kernel_initializer="he_normal",
)(neural_inputs)
 neural_inputs = keras.layers.Dropout(
 rate=hp.Float("dropout_2", min_value=0.1, max_value=0.9, default=0.2, step=0.1)
 )(neural_inputs)
 out = keras.layers.Dense(1, activation="sigmoid", )(neural_inputs)
 model = keras.models.Model(
 inputs=[input_word_ids, input_mask, segment_ids], outputs=out 
)
 model.compile(
 keras.optimizers.Adam(hp.Choice("learning_rate", [5e-5, 3e-5, 1e-5])) ,
loss=keras.losses.binary_crossentropy,
metrics=["accuracy"], 
)
 return model
to_remove = []
for l in [hh_ordered, hh_bool, hh_cont, ind_bool, ind_ordered, ind_cont]:
for c in l:
if c not in data:
to_remove.append(c )<drop_column>
tuner = Hyperband(
 build_model,
objective="val_accuracy",
max_epochs=3,
project_name=".. /output/kaggle/working/hyperparameter/hyperband",
)
for l in [hh_ordered, hh_bool, hh_cont, ind_bool, ind_ordered, ind_cont]:
for c in to_remove:
if c in l:
l.remove(c )<data_type_conversions>
x_train, x_val, y_train, y_val = train_test_split(
 df.clean_text, df.target, train_size=0.85
)
for variable in(hh_bool + ind_bool):
data[variable] = data[variable].astype('bool' )<data_type_conversions>
bert_xtrain = bert_encode(x_train)
 bert_xval = bert_encode(x_val )
for variable in(hh_cont + ind_cont):
data[variable] = data[variable].astype(float )<data_type_conversions>
bert_xtrain = np.asarray(bert_xtrain ).astype(np.float32)
 bert_xval = np.asarray(bert_xval ).astype(np.float32 )
for variable in(hh_ordered + ind_ordered):
try:
data[variable] = data[variable].astype(int)
 except Exception as e:
print(f'Could not convert {variable} because of missing values.' )<set_options>
if not already_trained:
tuner.search(
 {
"input_word_ids": bert_xtrain[0],
"input_mask": bert_xtrain[1],
"segment_ids": bert_xtrain[2],
},
y_train,
epochs=3,
batch_size=32,
validation_data=(
 {
"input_word_ids": bert_xval[0],
"input_mask": bert_xval[1],
"segment_ids": bert_xval[2],
},
y_val, 
),
callbacks=[checkpoint_cb, early_stopping_cb, tensorboard_cb], 
)
 else:
None
%matplotlib inline<create_dataframe>
tf.keras.utils.plot_model(bert_model, to_file="bert_model.png", dpi=96 )
es = ft.EntitySet(id = 'households')
 es.entity_from_dataframe(entity_id = 'data',
dataframe = data,
index = 'Id' )<categorify>
df_test = pd.read_csv(test_file)
 clean_text_column("text", df_test)
 bert_test = bert_encode(df_test.clean_text )
es.normalize_entity(base_entity_id='data',
new_entity_id='household',
index = 'idhogar',
additional_variables = hh_bool + hh_ordered + hh_cont + ['Target'])
 es<create_dataframe>
pred = bert_model.predict(bert_test )
feature_matrix, feature_names = ft.dfs(entityset=es,
target_entity = 'household',
max_depth = 2,
verbose = 1,
n_jobs = -1,
chunk_size = 100)
<drop_column>
round_predictions = pred.round().astype("int")
 df_submit = pd.read_csv(sample_submission_file)
 df_submit["target"] = round_predictions
df_submit.to_csv("submission.csv", index=False, header=True )
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 print(drop_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
feature_matrix.head()<drop_column>
round_predictions = pred.round().astype("int")
 df_submit = pd.read_csv(sample_submission_file)
 df_submit["target"] = round_predictions
df_submit.to_csv("submission.csv", index=False, header=True )
<split><EOS> 
threshold_predictions = pred.round().astype("int")
 df_submit = pd.read_csv(sample_submission_file)
 df_submit["target"] = round_predictions
df_submit.to_csv("submission.csv", index=False, header=True )