Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 20)
 pd.set_option('display.max_colwidth', -1 )
def get_test_loader(data_path=".. /input/tweet-sentiment-extraction/",
csv_name="test.csv",
max_seq_length=384,
model_type="bert-base-uncased",
batch_size=4,
num_workers=4):
CURR_PATH = ".. /input/"
csv_path = os.path.join(data_path, csv_name)
 df_test = pd.read_csv(csv_path)
 df_test.loc[:, "selected_text"] = df_test.text.values
df_test.loc[:, "cleaned_selected_text"] = df_test.text.values
if(model_type == "bert-base-uncased"):
tokenizer = BertTokenizer.from_pretrained(
 pretrained_model_name_or_path=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.txt".format(model_type)) ,
lowercase=True, 
)
 elif(model_type == "bert-large-uncased"):
tokenizer = BertTokenizer.from_pretrained(
 pretrained_model_name_or_path=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.txt".format(model_type)) ,
lowercase=True, 
)
 elif(model_type == "bert-base-cased"):
tokenizer = BertTokenizer.from_pretrained(
 pretrained_model_name_or_path=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.txt".format(model_type)) ,
lowercase=True, 
)
 elif(model_type == "bert-large-cased"):
tokenizer = BertTokenizer.from_pretrained(
 pretrained_model_name_or_path=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.txt".format(model_type)) ,
lowercase=True, 
)
 elif(model_type == "xlnet-base-cased")or(model_type == "xlnet-large-cased"):
tokenizer = XLNetTokenizer.from_pretrained(
 pretrained_model_name_or_path=os.path.join(CURR_PATH, "transformers-vocab/{}-spiece.model".format(model_type)) ,
lowercase=True, 
)
 elif(model_type == "albert-base-v2")or(model_type == "albert-large-v2")or(model_type == "albert-xlarge-v2"):
tokenizer = AlbertTokenizer.from_pretrained(
 pretrained_model_name_or_path=os.path.join(CURR_PATH, "transformers-vocab/{}-spiece.model".format(model_type)) ,
lowercase=True, 
)
 elif model_type == "roberta-base":
tokenizer = RobertaTokenizer(
 vocab_file=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.json".format(model_type)) ,
merges_file=os.path.join(CURR_PATH, "transformers-vocab/{}-merges.txt".format(model_type)) ,
lowercase=True, 
)
 elif model_type == "roberta-base-squad":
tokenizer = RobertaTokenizer(
 vocab_file=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.json".format(model_type)) ,
merges_file=os.path.join(CURR_PATH, "transformers-vocab/{}-merges.txt".format(model_type)) ,
lowercase=True, 
)
 elif model_type == "roberta-large":
tokenizer = RobertaTokenizer(
 vocab_file=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.json".format(model_type)) ,
merges_file=os.path.join(CURR_PATH, "transformers-vocab/{}-merges.txt".format(model_type)) ,
lowercase=True, 
)
 else:
raise NotImplementedError
ds_test = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.cleaned_selected_text.values,
old_selected_text=df_test.selected_text.values,
tokenizer=tokenizer,
model_type=model_type,
max_len=max_seq_length 
)
 loader = DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=num_workers, drop_last=False)
 return loader, tokenizer<load_pretrained>
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 print(len(df))
print(df.columns)
 df
class TweetBert(nn.Module):
def __init__(self, model_type="bert-large-uncased", hidden_layers=None):
super(TweetBert, self ).__init__()
self.model_name = 'TweetBert'
self.model_type = model_type
if hidden_layers is None:
hidden_layers = [-1]
self.hidden_layers = hidden_layers
if model_type == "bert-large-uncased":
bert_model_config = \
'.. /input/bertlargewholewordmaskingfinetunedsquad/bert-large-uncased-whole-word-masking-finetuned-squad-config.json'
self.config = BertConfig.from_json_file(bert_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
self.bert = BertModel(self.config)
 elif model_type == "bert-large-cased":
bert_model_config = \
'.. /input/bertlargewholewordmaskingfinetunedsquad/bert-large-cased-whole-word-masking-finetuned-squad-config.json'
self.config = BertConfig.from_json_file(bert_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
self.bert = BertModel(self.config)
 elif model_type == "bert-base-uncased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
self.config = BertConfig.from_json_file(bert_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
self.bert = BertModel(self.config)
 elif model_type == "bert-base-cased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-cased/bert_config.json'
self.config = BertConfig.from_json_file(bert_model_config)
 self.config.output_hidden_states = True
self.bert = BertModel(self.config)
 elif model_type == "roberta-base":
roberta_model_config = '.. /input/roberta-transformers-pytorch/roberta-base/config.json'
self.config = RobertaConfig.from_json_file(roberta_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
model_path = os.path.join('.. /input/roberta-transformers-pytorch/roberta-base/pytorch_model.bin')
 self.bert = RobertaModel.from_pretrained(model_path, config=self.config)
 elif model_type == "roberta-large":
roberta_model_config = '.. /input/roberta-transformers-pytorch/roberta-large/config.json'
self.config = RobertaConfig.from_json_file(roberta_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
model_path = os.path.join('.. /input/roberta-transformers-pytorch/roberta-large/pytorch_model.bin')
 self.bert = RobertaModel.from_pretrained(model_path, config=self.config)
 elif model_type == "albert-large-v2":
albert_model_config = '.. /input/pretrained-albert-pytorch/albert-large-v2/config.json'
self.config = AlbertConfig.from_json_file(albert_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
model_path = os.path.join('.. /input/pretrained-albert-pytorch/albert-large-v2/pytorch_model.bin')
 self.bert = AlbertModel.from_pretrained(model_path, config=self.config)
 elif model_type == "albert-base-v2":
albert_model_config = '.. /input/pretrained-albert-pytorch/albert-base-v2/config.json'
self.config = AlbertConfig.from_json_file(albert_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
model_path = os.path.join('.. /input/pretrained-albert-pytorch/albert-large-v2/pytorch_model.bin')
 self.bert = AlbertModel.from_pretrained(model_path, config=self.config)
 elif model_type == "xlnet-base-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-base-cased-config.json'
self.config = XLNetConfig.from_json_file(xlnet_model_config)
 self.config.output_hidden_states = True
self.config.hidden_dropout_prob = 0.1
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.bert = XLNetModel.from_pretrained(model_path, config=self.config)
 else:
raise NotImplementedError
weights_init = torch.zeros(len(hidden_layers)).float()
weights_init.data[:-1] = -3
self.layer_weights = torch.nn.Parameter(weights_init)
 self.qa_start_end = nn.Linear(self.config.hidden_size, 2)
 def init_weights_linear(m):
if type(m)== torch.nn.Linear:
torch.nn.init.normal_(m.weight, std=0.02)
 torch.nn.init.normal_(m.bias, 0)
 self.qa_start_end.apply(init_weights_linear)
 self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 def get_hidden_states(self, hidden_states):
fuse_hidden = None
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
fuse_hidden = hidden_states[hidden_layer].unsqueeze(-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer].unsqueeze(-1)
 fuse_hidden = torch.cat([fuse_hidden, hidden_state], dim=-1)
 fuse_hidden =(torch.softmax(self.layer_weights, dim=0)* fuse_hidden ).sum(-1)
 return fuse_hidden
def get_logits_by_random_dropout(self, fuse_hidden, fc):
logit = None
h = fuse_hidden
for j, dropout in enumerate(self.dropouts):
if j == 0:
logit = fc(dropout(h))
else:
logit += fc(dropout(h))
return logit / len(self.dropouts)
 def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
start_positions=None,
end_positions=None, 
):
if self.model_type == "roberta-base" or self.model_type == "roberta-base-squad" \
or self.model_type == "roberta-large":
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 hidden_states = outputs[2]
elif self.model_type == "xlnet-base-cased":
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids, 
)
 hidden_states = outputs[1]
else:
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 hidden_states = outputs[2]
fuse_hidden = self.get_hidden_states(hidden_states)
 fuse_hidden_context = fuse_hidden
if self.model_type == "xlnet-base-cased":
hidden_classification = fuse_hidden[:, -1, :]
else:
hidden_classification = fuse_hidden[:, 0, :]
logits = self.get_logits_by_random_dropout(fuse_hidden_context, self.qa_start_end)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits, end_logits = start_logits.squeeze(-1), end_logits.squeeze(-1)
 outputs =(start_logits, end_logits,)+ outputs[2:]
return outputs<load_pretrained>
def clean_text(text):
text = re.sub(r'http\S+', '', text)
 text = re.sub(r"(?:\@)\w+", '', text)
 text = re.sub(r'[^a-zA-Z0-9'.,?$&\s]', '', text)
 text = text.lower()
return text
for i in range(10):
index = np.random.randint(low=0, high=len(df))
print('Raw text:', df['text'][index])
 print('Cleaned text:', clean_text(df['text'][index]))
print('Label: ', df['target'][index], '
' )
def load_check_point(model, checkpoint_path, skip_layers=[]):
checkpoint_to_load = torch.load(checkpoint_path)
 model_state_dict = checkpoint_to_load['model']
state_dict = model.state_dict()
keys = list(state_dict.keys())
 for key in keys:
if any(s in key for s in skip_layers):
continue
try:
state_dict[key] = model_state_dict[key]
except:
print("Missing key:", key)
 model.load_state_dict(state_dict)
 return model<train_model>
def convert_to_features(data, tokenizer, max_len=None):
data = data.replace('
', '')
 if max_len is not None:
tokenized = tokenizer.encode_plus(
 data,
padding ='max_length',
max_length=max_len,
truncation=True,
return_tensors='np',
return_attention_mask=True,
return_token_type_ids=True, 
)
 else:
tokenized = tokenizer.encode_plus(
 data,
return_tensors='np',
return_attention_mask=True,
return_token_type_ids=True, 
)
 return tokenized
def create_inputs_with_targets(x, y, tokenizer, max_len=128):
dataset_dict = {
"input_ids": [],
"attention_mask": [],
'labels': []
}
for sentence, label in tqdm(zip(x,y)) :
cleaned_sentence = clean_text(sentence)
 temp = convert_to_features(cleaned_sentence, tokenizer, max_len=max_len)
 dataset_dict["input_ids"].append(temp["input_ids"][0])
 dataset_dict["attention_mask"].append(temp["attention_mask"][0])
 dataset_dict["labels"].append(label)
 x = [
np.array(dataset_dict["input_ids"]),
np.array(dataset_dict["attention_mask"]),
]
y = np.array(dataset_dict['labels'])
 return x, y
def create_inputs_without_targets(x, tokenizer, max_len=128):
dataset_dict = {
"input_ids": [],
"attention_mask": [],
}
for sentence in tqdm(x):
cleaned_sentence = clean_text(sentence)
 temp = convert_to_features(cleaned_sentence, tokenizer, max_len=max_len)
 dataset_dict["input_ids"].append(temp["input_ids"][0])
 dataset_dict["attention_mask"].append(temp["attention_mask"][0])
 x = [
np.array(dataset_dict["input_ids"]),
np.array(dataset_dict["attention_mask"]),
]
return x
def get_logits(model, checkpoint_path, folds, all_input_ids, all_attention_masks, all_token_type_ids=None):

 for model_idx, fold in enumerate(folds):
checkpoint = os.path.join(checkpoint_path, "fold_{}.pth".format(fold))
model = load_check_point(model, checkpoint, skip_layers=[])
 model.eval()
outputs = model(input_ids=all_input_ids, attention_mask=all_attention_masks,
token_type_ids=all_token_type_ids)
 if model_idx == 0:
start_logits, end_logits = torch.softmax(outputs[0], dim=-1)/ len(folds), torch.softmax(outputs[1], dim=-1)/ len(folds)
 else:
start_logits += torch.softmax(outputs[0], dim=-1)/ len(folds)
 end_logits += torch.softmax(outputs[1], dim=-1)/ len(folds)
 return start_logits.detach().cpu() , end_logits.detach().cpu()<load_pretrained>
base_model = 'bert-base-uncased'
bert_tokenizer = transformers.BertTokenizer.from_pretrained(base_model)
 max_len = 80
def get_word_level_logits(start_logits,
end_logits,
model_type,
tweet_offsets_word_level):
tweet_offsets_word_level = np.array(tweet_offsets_word_level)
 if model_type == "roberta-base" or model_type == "roberta-large" or model_type == "roberta-base-squad":
logit_offset = 4
elif(model_type == "albert-base-v2")or(model_type == "albert-large-v2")or(
 model_type == "albert-xlarge-v2"):
logit_offset = 3
elif(model_type == "xlnet-base-cased")or(model_type == "xlnet-large-cased"):
logit_offset = 2
elif(model_type == "bert-base-uncased")or(model_type == "bert-large-uncased"):
logit_offset = 3
elif(model_type == "bert-base-cased")or(model_type == "bert-large-cased"):
logit_offset = 3
prev = tweet_offsets_word_level[logit_offset]
word_level_bbx = []
curr_bbx = []
for i in range(len(tweet_offsets_word_level)- logit_offset - 1):
curr = tweet_offsets_word_level[i + logit_offset]
if curr[0] < prev[0] and curr[1] > prev[1]:
break
if curr[0] == prev[0] and curr[1] == prev[1]:
curr_bbx.append(i)
 else:
word_level_bbx.append(curr_bbx)
 curr_bbx = [i]
prev = curr
if len(word_level_bbx)== 0:
word_level_bbx.append(curr_bbx)
 for i in range(len(word_level_bbx)) :
word_level_bbx[i].append(word_level_bbx[i][-1] + 1)
 start_logits_word_level = [np.max(start_logits[bbx[0]: bbx[-1]])for bbx in word_level_bbx]
end_logits_word_level = [np.max(end_logits[bbx[0]: bbx[-1]])for bbx in word_level_bbx]
return start_logits_word_level, end_logits_word_level, word_level_bbx
def get_token_level_idx(start_logits,
end_logits,
start_logits_word_level,
end_logits_word_level,
word_level_bbx):
start_idx_word = np.argmax(start_logits_word_level)
 end_idx_word = np.argmax(end_logits_word_level)
 start_word_bbx = word_level_bbx[start_idx_word]
end_word_bbx = word_level_bbx[end_idx_word]
start_idx_in_word = np.argmax(start_logits[start_word_bbx[0]: start_word_bbx[-1]])
 end_idx_in_word = np.argmax(end_logits[end_word_bbx[0]: end_word_bbx[-1]])
 start_idx_token = start_word_bbx[start_idx_in_word]
end_idx_token = end_word_bbx[end_idx_in_word]
return np.argmax(start_logits), np.argmax(end_logits)
 def calculate_jaccard_score(
 original_tweet,
selected_text,
idx_start,
idx_end,
model_type,
tweet_offsets,
tokenizer,
sentiment):
if idx_end < idx_start:
filtered_output = original_tweet
else:
input_ids_orig = tokenizer.encode(original_tweet ).ids
input_ids = input_ids_orig + [2]
filtered_output = tokenizer.decode(input_ids[idx_start:idx_end+1])
 jac = 0
return jac, filtered_output<init_hyperparams>
validation_data_indices = df.sample(frac=0.2 ).index
validation_df = df.loc[validation_data_indices, :].reset_index(drop=True)
 train_df = df.drop(validation_data_indices, axis=0 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 x_train, y_train = create_inputs_with_targets(list(train_df['text']), list(train_df['target']), tokenizer=bert_tokenizer, max_len=max_len)
 x_val, y_val = create_inputs_with_targets(list(validation_df['text']), list(validation_df['target']), tokenizer=bert_tokenizer, max_len=max_len)
 x_test = create_inputs_without_targets(list(test_df['text']), tokenizer=bert_tokenizer, max_len=max_len)
 print('Training dataframe size: ', len(train_df))
print('Validation dataframe size: ', len(validation_df))
print('Test dataframe size: ', len(test_df))
def get_different_model_logits(model_type="roberta-base", \
hidden_layers=[-1, -2, -3, -4], \
checkpoint_path=".. /input/tweetrobertabase5fold42v8/", \
Config=None):

 if model_type == "roberta-base" or model_type == "roberta-large" or model_type == "roberta-base-squad":
offsets = 4
elif model_type == "albert-base-v2" or model_type == "albert-large-v2" or model_type == "albert-xlarge-v2":
offsets = 3
elif model_type == "xlnet-base-cased" or model_type == "xlnet-large-cased":
offsets = 2
elif model_type == "bert-base-uncased" or model_type == "bert-large-uncased" or model_type == "bert-base-cased" or model_type == "bert-large-cased":
offsets = 3
else:
raise NotImplementedError
loader, tokenizer = get_test_loader(data_path=".. /input/tweet-sentiment-extraction/",
max_seq_length=Config.max_seq_length,
model_type=model_type,
batch_size=Config.val_batch_size,
num_workers=Config.num_workers)
 model = TweetBert(model_type=model_type, hidden_layers=hidden_layers ).cuda().eval()
start_logits_token_level = []
end_logits_token_level = []
start_logits_word_level = []
end_logits_word_level = []
word_level_bbx = []
token_level_offsets = []
tweets = []
tweets_with_extra_spaces = []
sentiments = []
torch.cuda.empty_cache()
with torch.no_grad() :
for test_batch_i,(all_input_ids, all_attention_masks, all_token_type_ids,
all_start_positions, all_end_positions,
all_orig_tweet, all_orig_tweet_with_extra_space,
all_orig_selected, all_sentiment,
all_offsets_token_level, all_offsets_word_level)in enumerate(loader):
if(test_batch_i % 5 == 0):
print("Inferencing: ", test_batch_i, "of", len(loader))
all_input_ids = all_input_ids.cuda()
all_attention_masks = all_attention_masks.cuda()
all_token_type_ids = all_token_type_ids.cuda()
start_logits, end_logits = get_logits(model, \
checkpoint_path, \
[0, 1, 2, 3, 4], \
all_input_ids, \
all_attention_masks, \
all_token_type_ids)
 start_logits = start_logits[:, offsets:]
end_logits = end_logits[:, offsets:]
start_logits = torch.softmax(start_logits, dim=-1)
 end_logits = torch.softmax(end_logits, dim=-1)
 def to_numpy(tensor):
return tensor.numpy()
start_logits = to_numpy(start_logits)
 end_logits = to_numpy(( end_logits))
for px, tweet in enumerate(all_orig_tweet):
start_logits_word_level_sample, end_logits_word_level_sample, word_level_bbx_sample = get_word_level_logits(
 start_logits[px],
end_logits[px],
model_type,
all_offsets_word_level[px])
 start_logits_token_level.append(start_logits[px])
 end_logits_token_level.append(end_logits[px])
 start_logits_word_level.append(start_logits_word_level_sample)
 end_logits_word_level.append(end_logits_word_level_sample)
 word_level_bbx.append(word_level_bbx_sample)
 token_level_offsets.append(all_offsets_token_level[px])
 tweets.append(tweet)
 tweets_with_extra_spaces.append(all_orig_tweet_with_extra_space[px])
 sentiments.append(all_sentiment[px])
 return start_logits_token_level, end_logits_token_level, \
start_logits_word_level, end_logits_word_level, \
word_level_bbx, token_level_offsets, \
tweets, tweets_with_extra_spaces, sentiments, tokenizer<choose_model_class>
def create_model(model_name, max_len=128):
seed = 500
my_init = tf.keras.initializers.glorot_uniform(seed)
 max_len = max_len
encoder = transformers.TFAutoModel.from_pretrained(model_name)
 encoder.trainable = True
input_ids = keras.layers.Input(shape=(max_len,), dtype=tf.int32)
 attention_mask = keras.layers.Input(shape=(max_len,), dtype=tf.int32)
 sequence_output = encoder(input_ids, attention_mask=attention_mask)['last_hidden_state']
bi_lstm = tf.keras.layers.Bidirectional(keras.layers.LSTM(64, return_sequences=True))(sequence_output)
 avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
 max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
 concat = tf.keras.layers.concatenate([avg_pool, max_pool])
 dropout = tf.keras.layers.Dropout(0.3 )(concat)
 output = tf.keras.layers.Dense(1, activation="sigmoid" )(dropout)
 model = tf.keras.models.Model(
 inputs=[input_ids, attention_mask], outputs=[output] 
)
 return model
roberta_base_start_logits_token_level_42, roberta_base_end_logits_token_level_42, \
roberta_base_start_logits_word_level_42, roberta_base_end_logits_word_level_42, \
roberta_base_word_level_bbx, roberta_base_token_level_offsets, \
tweets, tweets_with_extra_spaces, sentiments, roberta_base_tokenizer = get_different_model_logits(model_type="roberta-base", \
hidden_layers=[-1, -2, -3, -4], \
checkpoint_path=".. /input/tweetrobertabasenewpipelinepreprocessing42/",\
Config=Config)
 roberta_base_start_logits_token_level_666, roberta_base_end_logits_token_level_666, \
roberta_base_start_logits_word_level_666, roberta_base_end_logits_word_level_666, \
_, _, _, _, _, _ = get_different_model_logits(model_type="roberta-base", \
hidden_layers=[-1, -2, -3, -4], \
checkpoint_path=".. /input/tweetrobertabasenewpipelinepreprocessing666/",\
Config=Config)
 roberta_base_start_logits_token_level_1234, roberta_base_end_logits_token_level_1234, \
roberta_base_start_logits_word_level_1234, roberta_base_end_logits_word_level_1234, \
_, _, _, _, _, _ = get_different_model_logits(model_type="roberta-base", \
hidden_layers=[-1, -2, -3, -4], \
checkpoint_path=".. /input/tweetrobertabasenewpipelinepreprocessing1234/",\
Config=Config )<define_variables>
epochs = 20
lr = 2e-4
use_tpu = True
if use_tpu:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 with strategy.scope() :
model = create_model(base_model, max_len=max_len)
 optimizer = keras.optimizers.Adam(learning_rate=lr)
 model.compile(optimizer=optimizer,
loss = keras.losses.BinaryCrossentropy() ,
metrics= [keras.metrics.BinaryAccuracy() ])
 else:
model = create_model()
model.summary()
sample_size = len(roberta_base_start_logits_token_level_42)
 roberta_base_start_logits_token_level = [(roberta_base_start_logits_token_level_42[i] +
roberta_base_start_logits_token_level_666[i] +
roberta_base_start_logits_token_level_1234[i] 
)/ 3 for i in range(sample_size)]
roberta_base_end_logits_token_level = [(roberta_base_end_logits_token_level_42[i] +
roberta_base_end_logits_token_level_666[i] +
roberta_base_end_logits_token_level_1234[i] 
)/ 3 for i in range(sample_size)]
roberta_base_start_logits_word_level = [(np.array(roberta_base_start_logits_word_level_42[i])+
np.array(roberta_base_start_logits_word_level_666[i])+
np.array(roberta_base_start_logits_word_level_1234[i])
)/ 3 for i in range(sample_size)]
roberta_base_end_logits_word_level = [(np.array(roberta_base_end_logits_word_level_42[i])+
np.array(roberta_base_end_logits_word_level_666[i])+
np.array(roberta_base_end_logits_word_level_1234[i])
)/ 3 for i in range(sample_size)] 
<define_search_model>
my_callbacks = [keras.callbacks.EarlyStopping(monitor='val_binary_accuracy', patience=2, mode='max', restore_best_weights=True)]
hist = model.fit(x_train,
y_train,
validation_data =(x_val, y_val),
epochs= epochs,
batch_size= 128,
callbacks = my_callbacks,
verbose= 1 )
_, _, albert_large_start_logits_word_level, albert_large_end_logits_word_level, _, _, _, _, _, _ = get_different_model_logits(model_type="albert-large-v2", \
hidden_layers=[-1, -2, -3, -4], \
checkpoint_path=".. /input/tweetalbertlargenewpipelinepreprocessingv1/",\
Config=Config )<define_search_model>
predictions = model.predict(x_test)
 ids = list(test_df['id'])
 target = [round(i[0])for i in predictions]
sub = pd.DataFrame({'id':ids, 'target':target}, index=None)
 sub.to_csv('submission.csv', index=False)
 sub
<define_variables><EOS>