Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<load_from_csv>
get_linear_schedule_with_warmup,
get_cosine_with_hard_restarts_schedule_with_warmup )
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )<drop_column>
copyfile(src = ".. /input/utils-v10/utilsv10.py", dst = ".. /working/utilsv10.py")
 copyfile(src = ".. /input/utils-v10/dataset10.py", dst = ".. /working/dataset10.py" )
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )<concatenate>
load_model, save_model, set_seed, write_event, evaluate, get_predicts_from_token_logits )
fe_name = list(test)
 df_train = train[fe_name]
df = pd.concat(( df_train,test))<import_modules>
from dataset10 import TrainDataset, MyCollator
from scipy import stats
from scipy.stats import norm<prepare_x_and_y>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' )
target = train['SalePrice']<create_dataframe>
tokenizer = AutoTokenizer.from_pretrained('.. /input/roberta-base/', do_lower_case=False )
null_df =(df.isna().sum() / len(df)) *100
null_df = null_df.drop(null_df[null_df == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :null_df})
 missing_data.head(20 )<feature_engineering>
class Args:
post = True
tokenizer = tokenizer
offset = 4
batch_size = 32
workers = 1
args = Args()
df["PoolQC"] = df["PoolQC"].fillna("None" )<feature_engineering>
collator = MyCollator()
test_set = TrainDataset(test, None, tokenizer=tokenizer, mode='test', offset=args.offset)
 test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, collate_fn=collator,
num_workers=args.workers )
df["MiscFeature"] = df["MiscFeature"].fillna("None" )<data_type_conversions>
class TweetModel(nn.Module):
def __init__(self, pretrain_path=None, dropout=0.2, config=None):
super(TweetModel, self ).__init__()
if config is not None:
self.bert = AutoModel.from_config(config)
 else:
config = AutoConfig.from_pretrained(pretrain_path, output_hidden_states=True)
 self.bert = AutoModel.from_pretrained(
 pretrain_path, cache_dir=None, config=config)
 self.cnn = nn.Conv1d(self.bert.config.hidden_size*3, self.bert.config.hidden_size, 3, padding=1)
 self.gelu = nn.GELU()
self.whole_head = nn.Sequential(OrderedDict([ 
('dropout', nn.Dropout(0.1)) , 
('l1', nn.Linear(self.bert.config.hidden_size*3, 256)) , 
('act1', nn.GELU()), 
('dropout', nn.Dropout(0.1)) , 
('l2', nn.Linear(256, 2))
]))
self.se_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.inst_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.dropout = nn.Dropout(0.1)
 def forward(self, inputs, masks, token_type_ids=None, input_emb=None):
_, pooled_output, hs = self.bert(
 inputs, masks, token_type_ids=token_type_ids, inputs_embeds=input_emb)
 seq_output = torch.cat([hs[-1],hs[-2],hs[-3]], dim=-1)
 avg_output = torch.sum(seq_output*masks.unsqueeze(-1), dim=1, keepdim=False)
 avg_output = avg_output/torch.sum(masks, dim=-1, keepdim=True)
 whole_out = self.whole_head(avg_output)
 seq_output = self.gelu(self.cnn(seq_output.permute(0,2,1)).permute(0,2,1))
se_out = self.se_head(self.dropout(seq_output))
inst_out = self.inst_head(self.dropout(seq_output))
return whole_out, se_out[:, :, 0], se_out[:, :, 1], inst_out
df["Alley"] = df["Alley"].fillna("None" )<data_type_conversions>
def predict(model: nn.Module, valid_df, valid_loader, args, progress=False)-> Dict[str, float]:
model.eval()
all_end_pred, all_whole_pred, all_start_pred, all_inst_out = [], [], [], []
if progress:
tq = tqdm.tqdm(total=len(valid_df))
with torch.no_grad() :
for tokens, types, masks, _, _, _, _, _, _, _ in valid_loader:
if progress:
batch_size = tokens.size(0)
 tq.update(batch_size)
 masks = masks.cuda()
tokens = tokens.cuda()
types = types.cuda()
whole_out, start_out, end_out, inst_out = model(tokens, masks, types)
 all_whole_pred.append(torch.softmax(whole_out, dim=-1)[:,1].cpu().numpy())
 inst_out = torch.softmax(inst_out, dim=-1)
 for idx in range(len(start_out)) :
length = torch.sum(masks[idx,:] ).item() -1
all_start_pred.append(torch.softmax(start_out[idx, args.offset:length], axis=-1 ).cpu())
 all_end_pred.append(torch.softmax(end_out[idx, args.offset:length], axis=-1 ).cpu())
 all_inst_out.append(inst_out[idx,:,1].cpu())
 assert all_start_pred[-1].dim() ==1
all_whole_pred = np.concatenate(all_whole_pred)
 if progress:
tq.close()
return all_whole_pred, all_start_pred, all_end_pred, all_inst_out
df["Fence"] = df["Fence"].fillna("None" )<feature_engineering>
config = RobertaConfig.from_pretrained('.. /input/roberta-base', output_hidden_states=True)
 model = TweetModel(config=config)
df["FireplaceQu"] = df["FireplaceQu"].fillna("None" )<groupby>
all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = [], [], [], []
for fold in range(10):
load_model(model, '.. /input/roberta-v10-10/best-model-%d.pt' % fold)
 model.cuda()
fold_whole_preds, fold_start_preds, fold_end_preds, fold_inst_preds = predict(model, test, test_loader, args, progress=True)
 all_whole_preds.append(fold_whole_preds)
 all_start_preds.append(fold_start_preds)
 all_end_preds.append(fold_end_preds)
 all_inst_preds.append(fold_inst_preds)
 all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = ensemble(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test)
 word_preds, inst_word_preds, scores = get_predicts_from_token_logits(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test, args)
 start_char_prob, end_char_prob = get_char_prob(all_start_preds, all_end_preds, test, args)
df["LotFrontage"] = df.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
test['start_char_prob'] = start_char_prob
test['end_char_prob'] = end_char_prob
<feature_engineering><EOS> 
test['selected_text'] = word_preds
def f(selected):
return " ".join(set(selected.lower().split()))
test[['textID','selected_text']].to_csv('submission.csv', index=False )