Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<predict_on_test>
!pip install -q transformers ekphrasis keras-tuner
def predicting(X_test, df_model, stop=9999999):
drop_list = ["building_id", "meter", "row_id"]
ids = np.sort(X_test["building_id"].unique())
 submission = pd.DataFrame(columns=["row_id", "meter_reading"])
 for bid in ids:
if stop < bid:
break
meters = np.sort(X_test[X_test["building_id"] == bid]["meter"].unique())
 for meter in meters:
Predecting building_id:{}, meter:{}
---------
".format(bid, meter))
X_each = X_test[(X_test["building_id"] == bid)&(X_test["meter"] == meter)]
row_ids = X_each.row_id
X_each.drop(drop_list, axis=1, inplace=True)
 model_half_1 = df_model[(df_model["building_id"] == bid)&(df_model["meter"] == meter)]["half_1"].values[0]
model_half_2 = df_model[(df_model["building_id"] == bid)&(df_model["meter"] == meter)]["half_2"].values[0]
pred = np.expm1(model_half_1.predict(X_each, num_iteration=model_half_1.best_iteration)) / 2
pred += np.expm1(model_half_2.predict(X_each, num_iteration=model_half_2.best_iteration)) / 2
sub_each = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission = pd.concat([submission, sub_each])
 return submission.sort_values("row_id" )<predict_on_test>
Input,
Dense,
Embedding,
Flatten,
Dropout,
GlobalMaxPooling1D,
GRU,
concatenate, 
)
 DistilBertTokenizerFast,
TFDistilBertModel,
DistilBertConfig, 
)
submission = predicting(X_test, df_model)
 submission<save_to_csv>
def print_metrics(model, x_train, y_train, x_val, y_val):
train_acc = dict(model.evaluate(x_train, y_train, verbose=0, return_dict=True)) [
"accuracy"
]
val_acc = dict(model.evaluate(x_val, y_val, verbose=0, return_dict=True)) [
"accuracy"
]
val_preds = model.predict(x_val)
 val_preds_bool = val_preds >= 0.5
print("")
 print(f"Training Accuracy: {train_acc:.2%}")
 print(f"Validation Accuracy: {val_acc:.2%}")
 print("")
 print(f"Validation f1 score: {sklearn.metrics.f1_score(val_preds_bool, y_val):.2%}" )
submission.to_csv("submission.csv", index=False )<set_options>
model_class, tokenizer_class, pretrained_weights =(TFDistilBertModel, DistilBertTokenizerFast, 'distilbert-base-uncased')
 pretrained_bert_tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
 def get_pretrained_bert_model(config=pretrained_weights):
if not config:
config = DistilBertConfig(num_labels=2)
 return model_class.from_pretrained(pretrained_weights, config=config)
py.init_notebook_mode(connected=True)
<set_options>
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
print("label counts:")
 train_df.target.value_counts()
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<feature_engineering>
print("train precentage of nulls:")
 print(round(train_df.isnull().sum() / train_df.count() * 100, 2))
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]<count_values>
print("test precentage of nulls:")
 print(round(test_df.isnull().sum() / test_df.count() * 100, 2))
leak_df.meter.value_counts()<count_duplicates>
classes = np.unique(train_df["target"])
 class_weights = sklearn.utils.class_weight.compute_class_weight(
 "balanced", classes=classes, y=train_df["target"] 
)
 class_weights = {clazz : weight for clazz, weight in zip(classes, class_weights)}
print(leak_df.duplicated().sum() )<feature_engineering>
train_df.drop_duplicates(subset="text", inplace=True, keep=False)
 print("train rows:", len(train_df.index))
print("test rows:", len(test_df.index))
print(len(leak_df)/ len(train_df))<set_options>
class TweetPreProcessor:

 def __init__(self):
self.text_processor = TextPreProcessor(
 normalize=[
"url",
"email",
"phone",
"user",
"time",
"date",
],
annotate={"repeated", "elongated"},
segmenter="twitter",
spell_correction=True,
corrector="twitter",
unpack_hashtags=False,
unpack_contractions=False,
spell_correct_elong=True,
fix_bad_unicode=True,
tokenizer=Tokenizer(lowercase=True ).tokenize,
dicts=[emoticons, slangdict], 
)
 def preprocess_tweet(self, tweet):
return " ".join(self.text_processor.pre_process_doc(tweet))
def __call__(self, tweet):
return self.text_processor.pre_process_doc(tweet)
 tweet_preprocessor = TweetPreProcessor()
del train_df
gc.collect()<load_from_csv>
for tweet in train_df[100:120]["text"]:
print("original: ", tweet)
 print("processed: ", tweet_preprocessor.preprocess_tweet(tweet))
print("" )
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0 )<feature_engineering>
train_df["text"] = train_df["text"].apply(tweet_preprocessor.preprocess_tweet)
 test_df["text"] = test_df["text"].apply(tweet_preprocessor.preprocess_tweet )
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )<merge>
train_df["keyword"].fillna("", inplace=True)
 test_df["keyword"].fillna("", inplace=True)
 train_df["keyword"] = train_df["keyword"].apply(urllib.parse.unquote)
 test_df["keyword"] = test_df["keyword"].apply(urllib.parse.unquote )
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<feature_engineering>
x_train, x_val, y_train, y_val = sklearn.model_selection.train_test_split(
 train_df[["text", "keyword"]], train_df["target"], test_size=0.3, random_state=42, stratify=train_df["target"]
)
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )<filter>
def tokenize_encode(tweets, max_length=None):
return pretrained_bert_tokenizer(
 tweets,
add_special_tokens=True,
truncation=True,
padding="max_length",
max_length=max_length,
return_tensors="tf", 
)
 max_length_tweet = 72
max_length_keyword = 8
train_tweets_encoded = tokenize_encode(x_train["text"].to_list() , max_length_tweet)
 validation_tweets_encoded = tokenize_encode(x_val["text"].to_list() , max_length_tweet)
 train_keywords_encoded = tokenize_encode(x_train["keyword"].to_list() , max_length_keyword)
 validation_keywords_encoded = tokenize_encode(x_val["keyword"].to_list() , max_length_keyword)
 train_inputs_encoded = dict(train_tweets_encoded)
 train_inputs_encoded["keywords"] = train_keywords_encoded["input_ids"]
validation_inputs_encoded = dict(validation_tweets_encoded)
 validation_inputs_encoded["keywords"] = validation_keywords_encoded["input_ids"]
leak_df[leak_df.pred1_l1p.isnull() ]<compute_train_metric>
train_dataset = tf.data.Dataset.from_tensor_slices(
(dict(train_tweets_encoded), y_train)
)
 val_dataset = tf.data.Dataset.from_tensor_slices(
(dict(validation_tweets_encoded), y_val)
)
 train_multi_input_dataset = tf.data.Dataset.from_tensor_slices(
(train_inputs_encoded, y_train)
)
 val_multi_input_dataset = tf.data.Dataset.from_tensor_slices(
(validation_inputs_encoded, y_val)
)
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )<compute_test_metric>
tfidf_vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(
 tokenizer=tweet_preprocessor, min_df=1, ngram_range=(1, 1), norm="l2" 
)
 train_vectors = tfidf_vectorizer.fit_transform(raw_documents=x_train["text"] ).toarray()
validation_vectors = tfidf_vectorizer.transform(x_val["text"] ).toarray()
N = 10
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p * leak_df['pred1'].values +(1.-p)* leak_df ['pred3'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
logisticRegressionClf = LogisticRegression(n_jobs=-1, C=2.78)
 logisticRegressionClf.fit(train_vectors, y_train)
 def print_metrics_sk(clf, x_train, y_train, x_val, y_val):
print(f"Train Accuracy: {clf.score(x_train, y_train):.2%}")
 print(f"Validation Accuracy: {clf.score(x_val, y_val):.2%}")
 print("")
 print(f"f1 score: {sklearn.metrics.f1_score(y_val, clf.predict(x_val)) :.2%}")
 print_metrics_sk(logisticRegressionClf, train_vectors, y_train, validation_vectors, y_val )
best_weight = np.argmin(scores)* 1./N
print(scores.min() , best_weight )<compute_test_metric>
feature_extractor = get_pretrained_bert_model()
model_outputs = feature_extractor.predict(
 train_dataset.batch(32)
)
 train_sentence_vectors = model_outputs.last_hidden_state[:, 0, :]
train_word_vectors = model_outputs.last_hidden_state[:, 1:, :]
model_outputs = feature_extractor.predict(
 val_dataset.batch(32)
)
 validation_sentence_vectors = model_outputs.last_hidden_state[:, 0, :]
validation_word_vectors = model_outputs.last_hidden_state[:, 1:, :]
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p *(best_weight * leak_df['pred1'].values +(1.-best_weight)* leak_df ['pred3'].values)+(1.-p)* leak_df ['pred2'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
logisticRegressionClf = LogisticRegression(n_jobs=-1, class_weight=class_weights)
 logisticRegressionClf.fit(train_sentence_vectors, y_train)
 print_metrics_sk(
 logisticRegressionClf,
train_sentence_vectors,
y_train,
validation_sentence_vectors,
y_val,
)
best_weight2 = np.argmin(scores)* 1./N
print(scores.min() , best_weight2)
<define_search_space>
def create_gru_model() -> keras.Model:
model = keras.Sequential()
model.add(keras.layers.InputLayer(input_shape=train_word_vectors.shape[1:]))
model.add(GRU(32, return_sequences=True))
model.add(GlobalMaxPooling1D())
 model.add(Dense(1, activation="sigmoid"))
model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_gru_model()
history = model.fit(
 train_word_vectors,
y_train,
validation_data=(validation_word_vectors, y_val),
class_weight=class_weights,
epochs=20,
verbose=0,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(model, train_word_vectors, y_train, validation_word_vectors, y_val )
all_combinations = list(np.linspace(0.2,0.5,31))
all_combinations<import_modules>
def create_multi_input_model() -> keras.Model:
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=feature_extractor.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(1 )(keyword_features)
 tweet_classification_vectors = keras.Input(( train_sentence_vectors.shape[1],), name="tweets")
 tweet_features = Dense(1, activation='relu' )(tweet_classification_vectors)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, tweet_classification_vectors], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_multi_input_model()
train_inputs = {"keywords" : train_keywords_encoded["input_ids"], "tweets" : train_sentence_vectors}
validation_inputs = {"keywords" : validation_keywords_encoded["input_ids"], "tweets" : validation_sentence_vectors}
history = model.fit(
 train_inputs,
y_train,
validation_data=(validation_inputs, y_val),
class_weight=class_weights,
epochs=20,
verbose=0,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(model, train_inputs, y_train, validation_inputs, y_val )
import itertools<concatenate>
def create_multi_input_rnn_model() -> keras.Model:
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=feature_extractor.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(1 )(keyword_features)
 tweet_token_embeddings = Input(train_word_vectors.shape[1:], name="tweets")
 tweet_features = GRU(32, return_sequences=True )(tweet_token_embeddings)
 tweet_features = GlobalMaxPooling1D()(tweet_features)
 tweet_features = Dense(1, activation='relu' )(tweet_features)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, tweet_token_embeddings], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_multi_input_rnn_model()
train_inputs = {"keywords" : train_keywords_encoded["input_ids"], "tweets" : train_word_vectors}
validation_inputs = {"keywords" : validation_keywords_encoded["input_ids"], "tweets" : validation_word_vectors}
history = model.fit(
 train_inputs,
y_train,
validation_data=(validation_inputs, y_val),
class_weight=class_weights,
epochs=20,
verbose=0,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(model, train_inputs, y_train, validation_inputs, y_val )
l = [all_combinations, all_combinations, all_combinations]
all_l = list(itertools.product(*l)) + list(itertools.product(*reversed(l)) )<define_variables>
def create_candidate_model_with_fx(hp: kerastuner.HyperParameters)-> keras.Model:
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=feature_extractor.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(hp.Choice("keyword_units", values=[1, 8, 16, 32], default=1))(keyword_features)
 tweet_token_embeddings = Input(train_word_vectors.shape[1:], name="tweets")
 tweet_features = GRU(hp.Choice("GRU_units", values=[8, 16, 32, 64, 128], default=32), return_sequences=True )(tweet_token_embeddings)
 tweet_features = Dropout(hp.Float("GRU_dropout", min_value=0.0, max_value=0.5, step=0.1))(tweet_features)
 tweet_features = GlobalMaxPooling1D()(tweet_features)
 for i in range(hp.Int("num_layers", min_value=0, max_value=3, step=1)) :
tweet_features = Dense(hp.Choice("layer_" + str(i)+ "_units", values=[2, 8, 16, 32, 64, 128, 256]), activation="relu" )(tweet_features)
 tweet_features = Dropout(hp.Float("layer_" + str(i)+ "_dropout", min_value=0.0, max_value=0.5, step=0.1))(tweet_features)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, tweet_token_embeddings], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
train_inputs = {"keywords" : train_keywords_encoded["input_ids"], "tweets" : train_word_vectors}
validation_inputs = {"keywords" : validation_keywords_encoded["input_ids"], "tweets" : validation_word_vectors}
filtered_combis = [l for l in all_l if l[0] + l[1] + l[2] > 0.95 and l[0] + l[1] + l[2] < 1.05]<compute_test_metric>
MAX_EPOCHS = 10
FACTOR = 3
ITERATIONS = 3
print(f"Number of models in each bracket: {math.ceil(1 + math.log(MAX_EPOCHS, FACTOR)) }")
 print(f"Number of epochs over all trials: {round(ITERATIONS *(MAX_EPOCHS *(math.log(MAX_EPOCHS, FACTOR)** 2)))}" )
best_combi = []
for i, combi in enumerate(filtered_combis):
score1 = combi[0]
score2 = combi[1]
score3 = combi[2]
v = score1 * leak_df['pred1'].values + score2 * leak_df['pred3'].values + score3 * leak_df['pred2'].values
vl1p = np.log1p(v)
 curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
if best_combi:
prev_score = best_combi[0][1]
if curr_score < prev_score:
best_combi[:] = []
best_combi += [(i, curr_score)]
else:
best_combi += [(i, curr_score)]
score = best_combi[0][1]
print(score )<feature_engineering>
tuner = kerastuner.Hyperband(
 create_candidate_model_with_fx,
max_epochs=MAX_EPOCHS,
hyperband_iterations=ITERATIONS,
factor=FACTOR,
objective="val_accuracy",
directory="hyperparam-search",
project_name="architecture-hyperband", 
)
 tuner.search(
 train_inputs,
y_train,
validation_data=(validation_inputs, y_val),
class_weight=class_weights,
epochs=10,
verbose=1,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=3,
restore_best_weights=True, 
)
 ], 
)
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
final_combi = filtered_combis[best_combi[0][0]]
w1 = final_combi[0]
w2 = final_combi[1]
w3 = final_combi[2]
print("The weights are: w1=" + str(w1)+ ", w2=" + str(w2)+ ", w3=" + str(w3))
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred3 + w3 * test_df.pred2
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<feature_engineering>
best_model = tuner.get_best_models() [0]
print("")
 best_arch_hp = tuner.get_best_hyperparameters() [0]
pprint.pprint(best_arch_hp.values, indent=4)
 print("")
 print_metrics(best_model, train_inputs, y_train, validation_inputs, y_val )
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<save_to_csv>
def create_bert_simple_for_ft() :
input_ids = Input(shape=(max_length_tweet,), dtype="int32", name="input_ids")
 attention_mask = Input(shape=(max_length_tweet,), dtype="int32", name="attention_mask")
 pretrained_bert_model = get_pretrained_bert_model()
bert_outputs = pretrained_bert_model(input_ids, attention_mask)
 prediction = Dense(1, activation="sigmoid" )(bert_outputs.last_hidden_state[:, 0, :])
 return keras.Model(inputs=[input_ids, attention_mask], outputs=prediction)
 model = create_bert_simple_for_ft()
model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=5e-5),
loss="binary_crossentropy",
metrics=["accuracy"], 
)
 model.fit(
 train_dataset.batch(32),
validation_data=val_dataset.batch(32),
class_weight=class_weights,
epochs=20,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(
 model, dict(train_tweets_encoded), y_train, dict(validation_tweets_encoded), y_val 
)
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<load_from_csv>
def create_bert_rnn_for_ft() :
pretrained_bert_model = get_pretrained_bert_model()
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=pretrained_bert_model.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(1 )(keyword_features)
 input_ids = Input(shape=(max_length_tweet,), dtype="int32", name="input_ids")
 attention_mask = Input(shape=(max_length_tweet,), dtype="int32", name="attention_mask")
 bert_outputs = pretrained_bert_model(input_ids, attention_mask)
 bert_token_embeddings = bert_outputs.last_hidden_state[:, 1:, :]
tweet_features = GRU(32, return_sequences=True )(bert_token_embeddings)
 tweet_features = GlobalMaxPooling1D()(tweet_features)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, input_ids, attention_mask], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam(learning_rate=5e-5),
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_bert_rnn_for_ft()
model.fit(
 train_multi_input_dataset.batch(32),
validation_data=val_multi_input_dataset.batch(32),
epochs=20,
class_weight=class_weights,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=3,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(
 model, train_inputs_encoded, y_train, validation_inputs_encoded, y_val
)
register_matplotlib_converters()
sub = None
for dirname, _, filenames in os.walk('/kaggle/input/ultimaincercare/'):
for filename in filenames:
filename = os.path.join(dirname, filename)
 print(filename)
 if sub is None:
sub = pd.read_csv(filename)
 else:
sub.meter_reading += pd.read_csv(filename, usecols=['meter_reading'] ).meter_reading
sub.meter_reading = sub.meter_reading.clip(lower=0.0)/ len(filenames)
 sub.describe()<save_to_csv>
def create_model_candidate() -> keras.Model:
pretrained_bert_model = get_pretrained_bert_model()
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=pretrained_bert_model.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(best_arch_hp.get("keyword_units"))(keyword_features)
 input_ids = Input(shape=(max_length_tweet,), dtype="int32", name="input_ids")
 attention_mask = Input(shape=(max_length_tweet,), dtype="int32", name="attention_mask")
 bert_outputs = pretrained_bert_model(input_ids, attention_mask)
 bert_token_embeddings = bert_outputs.last_hidden_state[:, 1:, :]
tweet_features = GRU(best_arch_hp.get("GRU_units"), return_sequences=True )(bert_token_embeddings)
 tweet_features = Dropout(best_arch_hp.get("GRU_dropout"))(tweet_features)
 tweet_features = GlobalMaxPooling1D()(tweet_features)
 for i in range(best_arch_hp.get("num_layers")) :
tweet_features = Dense(best_arch_hp.get("layer_" + str(i)+ "_units"), activation="relu" )(tweet_features)
 tweet_features = Dropout(best_arch_hp.get("layer_" + str(i)+ "_dropout"))(tweet_features)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, input_ids, attention_mask], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam(learning_rate=5e-5),
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
sub.to_csv(f'submission-final-1.csv', index=False, float_format='%g' )<import_modules>
model = create_model_candidate()
history = model.fit(
 train_multi_input_dataset.batch(32),
validation_data=val_multi_input_dataset.batch(32),
epochs=6,
class_weight=class_weights,
callbacks=[
keras.callbacks.EarlyStopping(
 monitor="val_accuracy", restore_best_weights=True 
)
 ], 
)
 best_epoch = len(history.history["val_accuracy"])- 1
print_metrics(
 model, train_inputs_encoded, y_train, validation_inputs_encoded, y_val
)
FileLink('submission-final-1.csv' )<define_variables>
test_tweets_encoded = tokenize_encode(test_df["text"].to_list() , max_length_tweet)
 test_inputs_encoded = dict(test_tweets_encoded)
 test_dataset = tf.data.Dataset.from_tensor_slices(test_inputs_encoded)
 test_keywords_encoded = tokenize_encode(test_df["keyword"].to_list() , max_length_keyword)
 test_inputs_encoded["keywords"] = test_keywords_encoded["input_ids"]
test_multi_input_dataset = tf.data.Dataset.from_tensor_slices(test_inputs_encoded )
path_data = "/kaggle/input/ashrae-energy-prediction/"
path_train = path_data + "train.csv"
path_test = path_data + "test.csv"
path_building = path_data + "building_metadata.csv"
path_weather_train = path_data + "weather_train.csv"
path_weather_test = path_data + "weather_test.csv"
myfavouritenumber = 13
seed = myfavouritenumber<load_from_csv>
full_train_dataset = train_multi_input_dataset.concatenate(val_multi_input_dataset)
 model = create_model_candidate()
model.fit(
 full_train_dataset.batch(32),
epochs=best_epoch,
class_weight=class_weights,
)
<set_options><EOS> 
preds = np.squeeze(model.predict(test_multi_input_dataset.batch(32)))
 preds =(preds >= 0.5 ).astype(int)
 pd.DataFrame({"id": test_df.id, "target": preds} ).to_csv("submission.csv", index=False )