Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
!pip install tweet-preprocessor
warnings.filterwarnings('ignore' )<set_options>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<feature_engineering>
tf.random.set_seed(123)
 np.random.seed(123 )
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
start_time = time.time()
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return round(float(len(c)) /(len(a)+ len(b)- len(c)) , 4 )
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
results_jaccard = []
for index, row in train.iterrows() :
sentence1 = row.keyword
sentence2 = row.text
jaccard_score = jaccard(sentence1, sentence2)
 results_jaccard.append([sentence1, sentence2, jaccard_score] )
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
jaccard_score = pd.DataFrame(results_jaccard, columns=['keyword', 'text', 'jaccard_score'] )
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )<load_from_csv>
stopwords_en = stopwords.words('english' )
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
def unicode_to_ascii(s):
return ''.join(c for c in unicodedata.normalize('NFD', s)if unicodedata.category(c)!= 'Mn' )
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
twitter_p.set_options(twitter_p.OPT.URL)
 def preprocess_sentence(w):
w = twitter_p.clean(w)
 w = unicode_to_ascii(w.lower().strip())
 w = re.sub(r"([@
w = re.sub(r'[" "]+', " ", w)
 w = re.sub(r"[^a-zA-Z@
w = ' '.join([word for word in w.split(' ')if word not in stopwords_en])
 w = w.rstrip().strip()
return w
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<set_options>
train['text'] = train['text'].apply(func=preprocess_sentence)
 train['keyword'] = train['keyword'].apply(func=preprocess_sentence)
 print(train.head(10))
warnings.filterwarnings('ignore' )<set_options>
test['text'] = test['text'].apply(func=preprocess_sentence)
 test['keyword'] = test['keyword'].apply(func=preprocess_sentence)
 print(test.head(10))
cuda_yes = torch.cuda.is_available()
print('Cuda is available?', cuda_yes)
 device = torch.device("cuda:0" if cuda_yes else "cpu")
 print('Device:', device)
<set_options>
text_list = np.stack([*train['text'], *train['keyword'], *test['text'], *test['keyword']])
 tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')
 tokenizer.fit_on_texts(text_list)
 print(len(tokenizer.word_index))
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<define_variables>
glove = np.load('/kaggle/input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', allow_pickle=True)
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
data['sentiment'] = row.sentiment
if self.labeled:
start_idx, end_idx,labels = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
data['labels']=labels
data['selected_text']=row.selected_text
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 if row.sentiment=="positive":
sent=1
elif row.sentiment=="negative":
sent=2
else:
sent=3
len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 is not None and idx1 is not None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
labels = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 labels.append(sent)
 elif j==1:
labels.append(sent)
 else:
labels.append(0)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
labels = torch.tensor(labels)
 return start_idx, end_idx, labels
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
input_vocab_size = len(tokenizer.word_index)+ 3
d_model = 300
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True,num_labels=NUM_LABELS)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.num_labels = config.num_labels
self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask, labels=None):
sequence_output, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3],hs[-4]])
 x = torch.mean(x, 0)
 x_drop = self.dropout(x)
 x = self.fc(x_drop)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 logits = self.classifier(x_drop)
 outputs=(logits,start_logits, end_logits)
 if labels is not None:
loss_fct = nn.CrossEntropyLoss()
if attention_mask is not None:
active_loss = attention_mask.view(-1)== 1
active_logits = logits.view(-1, self.num_labels)
 active_labels = torch.where(
 active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index ).type_as(labels)
)
 loss = loss_fct(active_logits, active_labels)
 else:
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs<compute_test_metric>
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english" )
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
words = glove.keys()
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts )
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text,selected_text,start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 jaccard_approx = jaccard(true, pred)
 jaccard_true = jaccard(selected_text,pred)
 return jaccard_approx,jaccard_true 
<train_model>
def create_embedding_matrix(vectors, to_word_it, inp_vocab_size, d_m, lemma_dict):
no_in_vocab = []
matrix = np.random.uniform(low=-1, high=1, size=(inp_vocab_size, d_m))
unknown_vector = np.zeros(( d_m,), dtype=np.float32)- 1
for key, index in to_word_it:
word = key
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = key.lower()
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = key.upper()
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = key.capitalize()
try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = ps.stem(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = lc.stem(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
word = sb.stem(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
try:
word = lemma_dict[key]
matrix[index] = vectors[word]
continue
except KeyError:
''
if len(key)> 1:
word = correction(key)
 try:
matrix[index] = vectors[word]
continue
except KeyError:
''
try:
matrix[index] = vectors[word]
except KeyError:
matrix[index] = unknown_vector
no_in_vocab.append(word)
 del vectors
print("no_in_vocab size:", len(no_in_vocab))
print("embedding_matrix shape:", matrix.shape)
 return matrix, no_in_vocab
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.to(device)
 for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_approx_jaccard = 0.0
epoch_true_jaccard = 0.0
epoch_start_end_loss=0.0
epoch_ner_loss=0.0
tk0 = tqdm(dataloaders_dict[phase], total=len(dataloaders_dict[phase]))
for data in tk0:
ids = data['ids'].to(device)
 masks = data['masks'].to(device)
 tweet = data['tweet']
selected_text=data['selected_text']
sentiment = data['sentiment']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].to(device)
 end_idx = data['end_idx'].to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
ner_loss,logits,start_logits, end_logits = model(ids, masks,labels)
 start_end_loss = criterion(start_logits, end_logits, start_idx, end_idx)
 loss=ner_loss+start_end_loss
if phase == 'train':
loss.backward()
optimizer.step()
scheduler.step()
epoch_loss += loss.item() * len(ids)
 epoch_start_end_loss += start_end_loss.item() * len(ids)
 epoch_ner_loss += ner_loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_approx_score,jaccard_true_score = compute_jaccard_score(
 tweet[i],
selected_text[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_approx_jaccard += jaccard_approx_score
epoch_true_jaccard += jaccard_true_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_ner_loss = epoch_ner_loss / len(dataloaders_dict[phase].dataset)
 epoch_start_end_loss = epoch_start_end_loss / len(dataloaders_dict[phase].dataset)
 epoch_approx_jaccard = epoch_approx_jaccard / len(dataloaders_dict[phase].dataset)
 epoch_true_jaccard = epoch_true_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | NER_Loss: {:.4f} | Start_End_Loss: {:.4f} | Approx_Jaccard: {:.4f} | True_Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss,epoch_ner_loss,epoch_start_end_loss,epoch_approx_jaccard,epoch_true_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
print("Spacy NLP...")
 text_list = pd.concat([train['text'], test['text']])
 print(len(tokenizer.word_index))
nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
num_epochs = 3
batch_size = 32
gradient_accumulation_steps = 1
warmup_proportion=0.1
NUM_LABELS=4
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )<load_from_csv>
embedding_matrix, no_in_vocab = create_embedding_matrix(glove, tokenizer.word_index.items() , input_vocab_size, d_model, lemma_dict)
 del glove
<feature_engineering>
def inter_section(texts, keywords, niv):
niv = set(niv)
 text_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')
 text_tokenizer.fit_on_texts(np.stack([*texts, *keywords]))
vocab = set(text_tokenizer.word_index.keys())
 text_in_niv = vocab.intersection(niv)
 print("vocab:", len(vocab), len(text_in_niv), len(text_in_niv)/len(vocab))
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
model_dir = ".. /input/tweet-sentiment-roberta-pytorch/"
for fold in range(skf.n_splits):
model = TweetModel()
model.to(device)
 model.load_state_dict(torch.load(model_dir+f'roberta_fold{fold+1}.pth',map_location=torch.device(device)))
 model.eval()
models.append(model)
 for data in tqdm(test_loader):
ids = data['ids'].to(device)
 masks = data['masks'].to(device)
 tweet = data['tweet']
sentiment = data['sentiment']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[2], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if(start_pred > end_pred):
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
inter_section(test['text'], test['keyword'], no_in_vocab )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<set_options>
def incorrect_count(train_texts, test_texts, vocab):
vocab = set(vocab)
 wrong_words = []
for text in train_texts:
intersection = set(text.split() ).intersection(vocab)
 if len(intersection)>0:
wrong_words.extend(intersection)
 train_ww_count = np.asarray(Counter(wrong_words ).most_common())
 train_ww_count = np.concatenate([train_ww_count, np.asarray(['train']*len(train_ww_count)) [:, np.newaxis]], axis=-1)
 wrong_words = []
for text in test_texts:
intersection = set(text.split() ).intersection(vocab)
 if len(intersection)>0:
wrong_words.extend(intersection)
 test_ww_count = np.asarray(Counter(wrong_words ).most_common())
 test_ww_count = np.concatenate([test_ww_count, np.asarray(['test']*len(test_ww_count)) [:, np.newaxis]], axis=-1)
 ww_count = np.concatenate([train_ww_count, test_ww_count], axis=0)
 ww_count = pd.DataFrame(ww_count, columns=['wrong_text', 'count','set'])
 ww_count['count'] = ww_count['count'].astype('int')
 ww_count = ww_count.sort_values(by='count', ascending=False)
 print('head:
', ww_count.head())
 print('
count<2:', len(ww_count[ww_count['count']<2])/len(ww_count))
plt.figure(figsize=(6,36))
sns.barplot(x='count', y='wrong_text', hue='set', orient='h', data=ww_count.head(100))
plt.show()
%load_ext wurlitzer
!ls '.. /input/ashrae-energy-prediction/'
!ls '.'<install_modules>
incorrect_count(train['text'], test['text'], no_in_vocab )
!pip install -i https://test.pypi.org/simple/ litemort==0.1.18
print(litemort.__version__)
<define_variables>
num_texts =len_sentence(train, 'text' )
isMORT = True
isImplicitMerge = True
gbm='MORT' if isMORT else 'LGB'
use_ucf=True
nTargetMeter=4
data_root = '.. /input/ashrae-energy-prediction/'
print(f"====== ImplicitMerge={isImplicitMerge} gbm={gbm} ======
" )<categorify>
num_keyword = len_sentence(train, 'keyword' )
def LoadUCF(data_root):
ucf_root = '.. /input/ashrae-ucf-spider-and-eda-full-test-labels'
ucf_leak_df = pd.read_pickle(f'{ucf_root}/site0.pkl')
 ucf_leak_df['meter_reading'] = ucf_leak_df.meter_reading_scraped
ucf_leak_df.drop(['meter_reading_original', 'meter_reading_scraped'], axis=1, inplace=True)
 ucf_leak_df.fillna(0, inplace=True)
 ucf_leak_df.loc[ucf_leak_df.meter_reading < 0, 'meter_reading'] = 0
ucf_leak_df = ucf_leak_df[ucf_leak_df.timestamp.dt.year > 2016]
print(len(ucf_leak_df))
return ucf_leak_df
LoadUCF("")
 def ReplaceUCF() :
print('ReplaceUCF...... ')
 leak_score = 0
leak_df = LoadUCF(data_root)
 sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter == m), 'meter_reading'].values
v1 = temp_df[temp_df.meter == m].meter_reading.values
leak_score += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter == m), 'meter_reading'] = temp_df[
temp_df.meter == m].meter_reading.values
print('UCF score = ', np.sqrt(leak_score / len(leak_df)))
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 print(sample_submission.head(100),sample_submission.tail(100))<categorify>
def texts_to_sequences(byte):
char = str(byte, encoding='utf-8')
 sequences = tokenizer.texts_to_sequences([char])
 return np.reshape(sequences,(-1))
class Whether(object):
def __init__(self, source, data_root,params=None):
self.source = source
self.data_root = data_root
self.lag_day=[3,72]
self.lag_feat_list=[]
def TimeAlignment(self,weather_df):
print(f"TimeAlignment@{self.source}\tdf{weather_df.shape}...... ")
 weather_key = ['site_id', 'timestamp']
temp_skeleton = weather_df[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).\
sort_values(by=weather_key ).copy()
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])[
'air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
weather_df['offset'] = weather_df.site_id.map(site_ids_offsets)
 weather_df['timestamp_aligned'] =(weather_df.timestamp - pd.to_timedelta(weather_df.offset, unit='H'))
weather_df['timestamp'] = weather_df['timestamp_aligned']
del weather_df['timestamp_aligned']
return weather_df
def df(self):
weather_df = pd.read_csv(f'{self.data_root}/weather_{self.source}.csv', parse_dates=['timestamp'],
dtype={'site_id': np.uint8, 'air_temperature': np.float16,
'cloud_coverage': np.float16, 'dew_temperature': np.float16,'precip_depth_1_hr': np.float16})
 print(f"{weather_df.shape}
{weather_df.isna().sum() }")
 weather_df = self.TimeAlignment(weather_df)
 weather_df = weather_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
w_sum = weather_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 for days in self.lag_day:
f_list = self.add_lag_feature(weather_df, window=days)
 self.lag_feat_list.extend(f_list)
 print(f"weather_{self.source}={weather_df.shape} features={weather_df.columns}")
 return weather_df
def add_lag_feature(self,weather_df, window=3):
group_df = weather_df.groupby('site_id')
 feat_list=[]
cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
feat_list.append(f'{col}_mean_lag{window}')
 weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
if col=='air_temperature':
feat_list.append(f'{col}_max_lag{window}')
 weather_df[f'{col}_max_lag{window}'] = lag_max[col]
feat_list.append(f'{col}_min_lag{window}')
 weather_df[f'{col}_min_lag{window}'] = lag_min[col]
feat_list.append(f'{col}_std_lag{window}')
 weather_df[f'{col}_std_lag{window}'] = lag_std[col]
return feat_list<feature_engineering>
def text_encode(keyword, lang, target):
keyword = texts_to_sequences(keyword.numpy())
 lang = [len(tokenizer.word_index), *keyword, len(tokenizer.word_index)+ 1, *texts_to_sequences(lang.numpy()), len(tokenizer.word_index)+2]
return lang, target
class ASHRAE_data(object):
def __init__(self, source,data_root,building_meta_df,weather_df):
self.category_cols = ['building_id', 'site_id', 'primary_use']
self.source = source
self.data_root = data_root
self.building_meta_df = building_meta_df
self.weather_df = weather_df
feats_whether =[e for e in list(self.weather_df.columns)if e not in('site_id','offset', 'timestamp')]
self.feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median']+feats_whether
print(f"ASHRAE_data_{self.source}...... category={self.category_cols}
feature_cols={self.feature_cols}...... ")
 self.some_rows = None
self.df_base = self.Load_Processing()
self.df_base_shape = self.df_base.shape
print(f"ASHRAE_data_ df_base={self.df_base_shape}")
 def fit(cls, df):
pass
def data_X_y(self,target_meter):
feat_v0 = self.feature_cols + self.category_cols
feat_infos = {"categorical": self.category_cols}
train_df = self.df_base
print(f"{self.source}_X_y@{target_meter} df_base={train_df.shape}...... ")
 pkl_path = f'./_ashrae_{self.source}_T{target_meter}_{self.some_rows}_{"Mg"if isImplicitMerge else ""}_.pickle'
self.merge_infos = []
target_train_df = train_df[train_df['meter'] == target_meter]
print(f"target@{target_meter}={target_train_df.shape}")
 if isImplicitMerge:
building_site = self.building_meta_df[['building_id', 'site_id']]
target_train_df = target_train_df.merge(building_site, on='building_id', how='left')
 self.building_merge_ = self.building_meta_df[['building_id', 'primary_use', 'square_feet', 'year_built']]
feat_v0 = feat_v0 + ['timestamp']
feat_v1 = list(set(feat_v0 ).intersection(set(list(self.weather_df.columns))))
self.weather_df = self.weather_df[feat_v1]
self.merge_infos = [
{'on': ['site_id', 'timestamp'], 'dataset': self.weather_df, "desc": "weather"},
{'on': ['building_id'], 'dataset': self.building_merge_, "desc": "building",
"feat_info": feat_infos},
]
else:
target_train_df = target_train_df.merge(self.building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(self.weather_df, on=['site_id', 'timestamp'], how='left')
 feat_v1 = list(set(feat_v0 ).intersection(set(list(target_train_df.columns))))
X_train = target_train_df[feat_v1]
print(f"data_X__@{target_meter}={X_train.shape}\toriginal={target_train_df.shape}\tmerge={isImplicitMerge}")
 if(self.source == "train"):
y_train = target_train_df['meter_reading_log1p'].values
else:
y_train=None
del target_train_df
gc.collect()
return X_train, y_train
def Load_Processing(self):
df = pd.read_csv(f'{self.data_root}/{self.source}.csv', dtype={'building_id': np.uint16, 'meter': np.uint8},
parse_dates=['timestamp'])
 ucf_leak_df = LoadUCF(data_root)
 if self.source=="train":
df = df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 if use_ucf:
ucf_year = [2017, 2018]
if True:
print('delete all buildings site0 in 2016')
 bids = ucf_leak_df.building_id.unique()
df = df[df.building_id.isin(bids)== False]
ucf_leak_df = ucf_leak_df[ucf_leak_df.timestamp.dt.year.isin(ucf_year)]
df = pd.concat([df, ucf_leak_df])
 df.reset_index(inplace=True)
 if self.some_rows is not None:
df, _ = Mort_PickSamples(self.some_rows, df, None)
 print(f'====== Some Samples@{self.source}...data={df.shape}')
 df["hour"] = np.uint8(df["timestamp"].dt.hour)
 df["weekend"] = np.uint8(df["timestamp"].dt.weekday)
 df["month"] = np.uint8(df["timestamp"].dt.month)
 df["dayofweek"] = np.uint8(df["timestamp"].dt.dayofweek)
 if self.source == "train":
df['meter_reading_log1p'] = np.log1p(df['meter_reading'])
 df_group = df.groupby('building_id')['meter_reading_log1p']
self.building_mean = df_group.mean().astype(np.float16)
 self.building_median = df_group.median().astype(np.float16)
 self.building_min = df_group.min().astype(np.float16)
 self.building_max = df_group.max().astype(np.float16)
 self.building_std = df_group.std().astype(np.float16)
 print(f"building_mean={self.building_mean.head() }")
 else:
self.building_mean=train_datas.building_mean
print(f"test_datas.building_mean={self.building_mean.shape}")
 self.building_median=train_datas.building_median
self.building_min=train_datas.building_min
self.building_max=train_datas.building_max
self.building_std=train_datas.building_std
df['building_mean'] = df['building_id'].map(self.building_mean)
 df['building_median'] = df['building_id'].map(self.building_median)
 df['building_min'] = df['building_id'].map(self.building_min)
 df['building_max'] = df['building_id'].map(self.building_max)
 df['building_std'] = df['building_id'].map(self.building_std)
 return df<feature_engineering>
def tf_encode(id_num, keyword, lang, target):
lang, target = tf.py_function(
 text_encode,
[keyword, lang, target],
[tf.int64, tf.int64])
 id_num.set_shape(None)
 lang.set_shape([None])
 target.set_shape([])
 return id_num, lang, target
def LoadBuilding(data_root):
building_meta_df = pd.read_csv(f'{data_root}/building_metadata.csv')
 primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 print(f"{building_meta_df.shape}
{building_meta_df.head() }")
 return building_meta_df
building_meta_df=LoadBuilding(data_root)
 weather_test_df = Whether('test', data_root ).df()
print(f"weather_test_df={weather_test_df.shape} ")
 weather_train_df = Whether('train', data_root ).df()
print(f"weather_train_df={weather_train_df.shape}" )<init_hyperparams>
class EmbeddingLayer(object):
def __init__(self):
self.kernels = tf.Variable(initial_value=embedding_matrix, trainable=False, name='Embedding_kernels')
 def __call__(self, x):
embeddings = tf.nn.embedding_lookup(params=self.kernels, ids=x)
 return embeddings
early_stop = 20
verbose_eval = 5
metric = 'l2'
num_rounds = 1000; lr = 0.05; bf = 0.3
params = {'num_leaves': 31, 'n_estimators': num_rounds,
'objective': 'regression',
'max_bin': 256,
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric, "verbose_eval": verbose_eval, 'n_jobs': 8, "elitism": 0,"debug":'1',
"early_stopping_rounds": early_stop, "adaptive": 'weight1', 'verbose': 0, 'min_data_in_leaf': 200,
}
def fit_regressor(train, val,target_meter,fold, some_params, devices=(-1,), merge_info=None, cat_features=None):
t0=time.time()
X_train, y_train = train
X_valid, y_valid = val
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 if False:
col_y = pd.DataFrame(y_train)
 col_X = X_train.reset_index(drop=True)
 d_train = pd.concat([col_y, col_X], ignore_index=True, axis=1)
 np.savetxt("E:/2/LightGBM-master/examples/regression/case_cys_.csv", d_train, delimiter='\t')
 print("X_train={}, y_train={} d_train={}".format(col_X.shape, col_y.shape, d_train.shape))
if isMORT:
some_params['verbose']=666 if fold==0 else 0
merge_datas=[]
model = LiteMORT(some_params,merge_infos=merge_info)
 model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], categorical_feature=cat_features)
 fold_importance = None
log = ""
else:
some_params['verbose'] = 0
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB: parmas=',params)
 model = lgb.train(some_params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training'][metric], 'valid/mae': model.best_score['valid_1'][metric]}
y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 oof_loss = mean_squared_error(y_valid, y_pred_valid)
 print(f'METER:{target_meter} Fold:{fold} MSE: {oof_loss:.4f} time={time.time() - t0:.5g}', flush=True)
 return model, y_pred_valid, log<train_model>
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000,(2*(i//2)) /np.float32(d_model))
return pos * angle_rates
train_datas = ASHRAE_data("train",data_root,building_meta_df,weather_train_df)
 print(train_datas.building_mean.shape)
 print(train_datas.building_mean.head(5))<prepare_x_and_y>
def positional_encoding(postion, d_model):
angle_rads = get_angles(np.arange(postion)[:,np.newaxis], np.arange(d_model)[np.newaxis,:], d_model)
 angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
 angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
 pos_encoding = angle_rads[np.newaxis,...]
return tf.cast(pos_encoding, dtype=tf.float32 )
folds = 8
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 cat_features=None
meter_models=[]
losses=[]
for target_meter in range(nTargetMeter):
X_train, y_train = train_datas.data_X_y(target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print(f'target_meter={target_meter} X_train={X_train.shape}
features={X_train.columns}')
 cat_features = train_datas.category_cols
print('cat_features', cat_features)
 if False :
feat_select = X_train.columns
feat_select = list(set(feat_select)- set(feat_fix))
params['early_stopping_rounds'] = 50
params['category_features'] = cat_features
MORT_feat_select_(X_train, y_train, feat_fix, feat_select,params,nMostSelect=(int )(len(feat_select)/2))
input("...... MORT_feat_search...... ")
 sys.exit(-100)
 t0=time.time()
fold = 0
models_ = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx, :], y_train[train_idx]
valid_data = X_train.iloc[valid_idx, :], y_train[valid_idx]
params['seed'] = seed
print(f'fold={fold} train={train_data[0].shape},valid={valid_data[0].shape}')
 model, y_pred_valid, log = fit_regressor(train_data, valid_data,target_meter,fold,some_params=params,merge_info=train_datas.merge_infos, cat_features=cat_features)
 y_valid_pred_total[valid_idx] = y_pred_valid
models_.append(model)
 del train_data,valid_data
gc.collect()
fold=fold+1
meter_loss = mean_squared_error(y_train, y_valid_pred_total)
 print(f'======METER:{target_meter} MSE: {meter_loss:.4f} time={time.time() - t0:.5g}
')
 losses.append(meter_loss)
 meter_models.append(models_)
 del X_train, y_train
gc.collect()<predict_on_test>
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), dtype=tf.float32)
 return seq[:, tf.newaxis, tf.newaxis, :]
test_datas = ASHRAE_data("test",data_root,building_meta_df,weather_test_df)
 del train_datas
gc.collect()
test_df = test_datas.df_base
def pred(X_test, models, batch_size=1000000):
if isMORT and isImplicitMerge:
batch_size=batch_size*10
iterations =(X_test.shape[0] + batch_size -1)// batch_size
nSamp = X_test.shape[0]
print(f'iterations={iterations}\tnSamp={nSamp}\tbatch_size={batch_size}')
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
n_1=min(nSamp,(k+1)*batch_size)
 y_pred_test = model.predict(X_test[k*batch_size:n_1], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:n_1] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total<set_options>
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones(( size,size)) , -1, 0)
 return mask
warnings.filterwarnings('ignore')
<load_from_csv>
def scaled_dot_product_attention(q, k, v, mask):
matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(q)[-1], dtype=tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits +=(mask * -1e9)
 attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights
test = pd.read_csv('.. /input/ashrae-energy-prediction/test.csv', index_col=0, parse_dates = ['timestamp'])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv', usecols=['site_id', 'building_id'] )<merge>
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, num_heads, d_model):
super(MultiHeadAttention, self ).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % num_heads == 0
self.depth = d_model // num_heads
self.wq = tf.keras.layers.Dense(units=d_model)
 self.wk = tf.keras.layers.Dense(units=d_model)
 self.wv = tf.keras.layers.Dense(units=d_model)
 self.dense = tf.keras.layers.Dense(units=d_model)
 def split_heads(self, x, batch_size):
x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask)
 scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(scaled_attention,(batch_size, -1, self.d_model))
output = self.dense(concat_attention)
 return output, attention_weights
test = test.merge(building, left_on = "building_id", right_on = "building_id", how = "left" )<load_from_csv>
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(units=dff, activation='relu'),
tf.keras.layers.Dense(units=d_model)
 ] )
submission_base = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0 )<create_dataframe>
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate):
super(EncoderLayer, self ).__init__()
self.mha = MultiHeadAttention(num_heads=num_heads, d_model=d_model)
 self.ffn = point_wise_feed_forward_network(d_model=d_model, dff=dff)
 self.dropout1 = tf.keras.layers.Dropout(rate=rate)
 self.dropout2 = tf.keras.layers.Dropout(rate=rate)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 def call(self, x, training, mask):
attn_output, attn_weights = self.mha(x, x, x, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
submission = submission_base.copy()<load_from_csv>
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, maximum_position_encoding, rate=0.1):
super(Encoder, self ).__init__()
self.num_layers = num_layers
self.d_model = d_model
self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)
 self.enc_layers = [EncoderLayer(d_model=d_model, num_heads=num_heads, dff=dff, rate=rate)for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate=rate)
 def call(self, x, training, mask, positition=True):
seq_len = tf.shape(x)[1]
x *= tf.math.sqrt(tf.cast(self.d_model, dtype=tf.float32))
if positition:
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
 return x
site_0 = pd.read_csv('.. /input/new-ucf-starter-kernel/submission_ucf_replaced.csv', index_col=0)
 submission.loc[test[test['site_id']==0].index, 'meter_reading'] = site_0['meter_reading']
del site_0
gc.collect()<load_pretrained>
class OutputLayer(tf.keras.layers.Layer):
def __init__(self, units, rate):
super(OutputLayer, self ).__init__()
self.gapool1d = tf.keras.layers.GlobalAveragePooling1D()
self.dense = tf.keras.layers.Dense(units=units, activation='relu')
 self.final_layer = tf.keras.layers.Dense(units=2)
 self.dropout = tf.keras.layers.Dropout(rate=rate)
 def call(self, enc, training):
x = self.gapool1d(enc)
 x = self.dense(x)
 x = self.dropout(x, training=training)
 x = self.final_layer(x)
 return x
with open('.. /usr/lib/ucl_data_leakage_episode_2/site1.pkl', 'rb')as f:
site_1 = pickle.load(f)
 site_1 = site_1[site_1['timestamp'].dt.year > 2016]<merge>
class TransformerCategorical(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, maximum_position_encoding, output_units, rate=0.1):
super(TransformerCategorical, self ).__init__()
self.embedding = EmbeddingLayer()
self.encoder = Encoder(num_layers, d_model, num_heads, dff, maximum_position_encoding, rate)
 self.outputlayer = OutputLayer(output_units, rate)
 def call(self, lang, training, enc_padding_mask):
enc_input = self.embedding(lang)
 enc_output = self.encoder(enc_input, training, enc_padding_mask)
 out = self.outputlayer(enc_output, training)
 return out
t = test[['building_id', 'meter', 'timestamp']]
t['row_id'] = t.index
site_1 = site_1.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_1 = site_1[['meter_reading_scraped', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_1.index, 'meter_reading'] = site_1['meter_reading_scraped']
del site_1
gc.collect()<load_from_csv>
num_layers = 6
d_model = d_model
num_heads = 6
dff = 512
pe_input = input_vocab_size
output_units = 64
rate = 0.1
site_2 = pd.read_csv('.. /input/asu-buildings-energy-consumption/asu_2016-2018.csv', parse_dates = ['timestamp'])
 site_2 = site_2[site_2['timestamp'].dt.year > 2016]<merge>
tsfr_categorical = TransformerCategorical(num_layers, d_model, num_heads, dff, pe_input, output_units, rate )
t = test[['building_id', 'meter', 'timestamp']]
t['row_id'] = t.index
site_2 = site_2.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_2 = site_2[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_2.index, 'meter_reading'] = site_2['meter_reading']
del site_2
gc.collect()<load_from_csv>
class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, d_model, warmup_steps=600):
super(CustomSchedule, self ).__init__()
self.d_model = tf.cast(d_model, dtype=tf.float32)
 self.warmup_steps = warmup_steps
def __call__(self, step):
step = step + 100
arg1 = step ** -0.8
arg2 = step *(self.warmup_steps ** -1.5)
 return tf.math.rsqrt(self.d_model)* tf.math.minimum(arg1, arg2 )
site_4 = pd.read_csv('.. /input/ucb-data-leakage-site-4/site4.csv' )<save_to_csv>
learning_rate = CustomSchedule(d_model)
 optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9 )
submission.to_csv('submission.csv' )<set_options>
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)
 def loss_function(real, pred):
loss_ = loss_object(real, pred)
 return tf.reduce_mean(loss_)
 def acc_function(real, pred):
predictions = tf.math.argmax(pred, axis=1)
 predictions = tf.cast(predictions, dtype=tf.int64)
 accuracy = tf.cast(tf.math.equal(predictions, real), dtype=tf.float32)
 ave_acc = tf.reduce_mean(accuracy)
 return ave_acc
warnings.filterwarnings('ignore')
<load_from_csv>
train_step_signature = [
tf.TensorSpec(shape=(None, None), dtype=tf.int64),
tf.TensorSpec(shape=(None,), dtype=tf.int64)
 ]
test = pd.read_csv('.. /input/ashrae-energy-prediction/test.csv', index_col=0, parse_dates = ['timestamp'])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv', usecols=['site_id', 'building_id'] )<merge>
@tf.function(input_signature=train_step_signature)
 def train_step(lang, targ):
enc_padding_mask = create_padding_mask(lang)
 with tf.GradientTape() as tape:
predictions = tsfr_categorical(lang, True, enc_padding_mask)
 loss = loss_function(targ, predictions)
 loss_regularization = []
for w in tsfr_categorical.trainable_variables:
loss_regularization.append(tf.nn.l2_loss(w))
loss_regularization = tf.reduce_sum(tf.stack(loss_regularization))
loss = loss + 0.01 * loss_regularization
gradients = tape.gradient(loss, tsfr_categorical.trainable_variables)
 optimizer.apply_gradients(zip(gradients, tsfr_categorical.trainable_variables))
accuracy = acc_function(targ, predictions)
 return loss, accuracy
test = test.merge(building, left_on = "building_id", right_on = "building_id", how = "left")
 t = test[['building_id', 'meter', 'timestamp']]
t['row_id'] = t.index<load_from_csv>
@tf.function(input_signature=train_step_signature)
 def valid_step(lang, targ):
enc_padding_mask = create_padding_mask(lang)
 predictions = tsfr_categorical(lang, False, enc_padding_mask)
 loss = loss_function(targ, predictions)
 accuracy = acc_function(targ, predictions)
 return loss, accuracy
submission_base = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0 )<create_dataframe>
BATCH_SIZE = 2048
BUFFLE_SIZE = 8000
def data_generator(data):
dataset = tf.data.Dataset.from_tensor_slices(( data['id'], data['keyword'], data['text'], data['target']))
dataset = dataset.map(tf_encode)
 dataset = dataset.cache().shuffle(BUFFLE_SIZE ).padded_batch(BATCH_SIZE)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset
train_data = train.sample(frac=0.9)
 val_data = train[~train.index.isin(train_data.index)]
train_dataset = data_generator(train_data)
 val_dataset = data_generator(val_data)
 sample = next(iter(train_dataset))
print(sample )
submission = submission_base.copy()<load_from_csv>
Epochs = 100
site_0 = pd.read_csv('.. /input/new-ucf-starter-kernel/submission_ucf_replaced.csv', index_col=0)
 submission.loc[test[test['site_id']==0].index, 'meter_reading'] = site_0['meter_reading']
del site_0
gc.collect()<load_pretrained>
tensorboard = {'Train_loss':[],'Train_acc':[],'Val_loss':[],'Val_acc':[]}
for epoch in range(Epochs):
train_loss = []
train_accuracy = []
val_loss = []
val_accuracy = []
for _, lang, targ in train_dataset:
loss, acc = train_step(lang, targ)
 train_loss.append(loss)
 train_accuracy.append(acc)
 for _, lang, targ in val_dataset:
loss, acc = valid_step(lang, targ)
 val_loss.append(loss)
 val_accuracy.append(acc)
 ave_val_acc = np.mean(val_accuracy)
 if epoch !=0 and max(tensorboard['Val_acc'])<= ave_val_acc:
tsfr_categorical.save_weights('/kaggle/working/checkpoint/best_val')
 tensorboard['Train_loss'].append(np.mean(train_loss))
tensorboard['Train_acc'].append(np.mean(train_accuracy))
tensorboard['Val_loss'].append(np.mean(val_loss))
tensorboard['Val_acc'].append(ave_val_acc )
with open('.. /usr/lib/ucl_data_leakage_episode_2/site1.pkl', 'rb')as f:
site_1 = pickle.load(f)
 site_1 = site_1[site_1['timestamp'].dt.year > 2016]<merge>
for index in range(len(tensorboard['Train_loss'])) :
print(f"\033[0;34mEpoch\033[0m:{index}, Loss:{tensorboard['Train_loss'][index]}, Accuracy:{tensorboard['Train_acc'][index]}",
f"Valid_Loss:{tensorboard['Val_loss'][index]}, Valid_Accuracy:{tensorboard['Val_acc'][index]}" )
site_1 = site_1.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_1 = site_1[['meter_reading_scraped', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_1.index, 'meter_reading'] = site_1['meter_reading_scraped']
del site_1
gc.collect()<load_from_csv>
venv_target = np.array([0]*len(test['text']))
test_target = pd.read_csv('/kaggle/input/test-twitter/perfect_submission.csv')
 test_dataset = tf.data.Dataset.from_tensor_slices(( test['id'], test['keyword'], test['text'], test_target['target']))
test_dataset = test_dataset.map(tf_encode)
 test_dataset = test_dataset.padded_batch(BATCH_SIZE )
site_2 = pd.read_csv('.. /input/asu-buildings-energy-consumption/asu_2016-2018.csv', parse_dates = ['timestamp'])
 site_2 = site_2[site_2['timestamp'].dt.year > 2016]<merge>
evaluate = TransformerCategorical(num_layers, d_model, num_heads, dff, pe_input, output_units, rate)
 evaluate.load_weights('/kaggle/working/checkpoint/best_val' )
site_2 = site_2.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_2 = site_2[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_2.index, 'meter_reading'] = site_2['meter_reading']
del site_2
gc.collect()<feature_engineering>
results = []
for id_num, lang, _ in test_dataset:
enc_padding_mask = create_padding_mask(lang)
 predictions = evaluate(lang, False, enc_padding_mask)
 predictions = tf.math.argmax(predictions, axis=1)
 predictions = tf.cast(predictions, dtype=tf.int32)
 predictions = tf.reshape(predictions,(-1))
results.extend(zip(id_num.numpy() , predictions.numpy()))
site_4 = pd.read_csv('.. /input/ucb-data-leakage-site-4/site4.csv', parse_dates = ['timestamp'])
 site_4.columns = ['building_id', 'timestamp', 'meter_reading']
site_4['meter'] = 0
site_4['timestamp'] = pd.DatetimeIndex(site_4['timestamp'])+ timedelta(hours=-8)
 site_4 = site_4[site_4['timestamp'].dt.year > 2016]<merge>
label_equal = 0
for index, value in enumerate(results):
if value[1] == test_target['target'][index]:
label_equal +=1
print('result_score:', label_equal/len(results))
site_4 = site_4.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_4 = site_4[['meter_reading', 'row_id']].dropna().set_index('row_id')
 submission.loc[site_4.index, 'meter_reading'] = site_4['meter_reading']
del site_4
gc.collect()<save_to_csv>
submission.to_csv('/kaggle/working/submission.csv', index=False )
<define_variables><EOS> 
print(time.time() - start_time)
 gc.collect()