Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<set_options>
warnings.filterwarnings("ignore")
 warnings.filterwarnings("ignore",category=UserWarning )
pd.set_option('display.max_columns', 100)
 pd.set_option("display.max_rows", 400)
 random_state = 42<load_from_csv>
copyfile(src = ".. /input/utils-v10/utilsv10.py", dst = ".. /working/utilsv10.py")
 copyfile(src = ".. /input/utils-v10/dataset10.py", dst = ".. /working/dataset10.py")
 copyfile(src = ".. /input/utils-v10/dataset11.py", dst = ".. /working/dataset11.py")
 load_model, save_model, set_seed, write_event, evaluate, get_predicts_from_token_logits)
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 df_target = df_train[['SalePrice']]
df = df_train.append(df_test)
 df.drop(columns=['Id'], inplace=True )<feature_engineering>
df_pred = pd.read_csv('.. /input/tweet-sentiment-fast/test_all_post_finetune_0608.csv')
 df_pred1 = pd.read_csv('.. /input/tweet-sentiment-fast/test_all_post_finetune_large.csv')
 df_train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 df_test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 df_test.loc[:, 'selected_text'] = df_test.text.values
df_test['text_clean'] = df_test['text'].apply(lambda x: " ".join(x.split()))
df_full = pd.read_csv('/kaggle/input/complete-tweet-sentiment-extraction-data/tweet_dataset.csv')
 df_full = df_full[df_full.text.notnull() ].copy()
df_full['text_clean'] = df_full['text'].apply(lambda x: " ".join(x.split()))
df_full = df_full.drop_duplicates(subset='text')
 df_full = df_full[~df_full.aux_id.isin(df_train.textID)]
df_full.rename(columns={'sentiment': 'raw_sentiment'}, inplace=True )
def mount_df_info(df):
df_info = df.dtypes.to_frame(name='type')
 df_info['count_null'] = df.isnull().values.sum(axis=0)
 df_info['nunique'] = df.nunique().values
df_info['count_zeros'] =(df == 0 ).values.sum(axis=0)
 df_info['max_value'] = df.max()
df_info['min_value'] = df.min()
return df_info<sort_values>
def find_sentiment(textID, text, sentiment):
text_clean = " ".join(text.split())
 if textID in df_full.aux_id.values:
return df_full['raw_sentiment'].loc[df_full.aux_id==textID].values[0]
elif text in df_full.text.values:
return df_full['raw_sentiment'].loc[df_full.text==text].values[0]
elif text_clean in df_full.text_clean.values:
return df_full['raw_sentiment'].loc[df_full.text_clean==text_clean].values[0]
else:
return sentiment
df_info = mount_df_info(df)
 df_info.sort_values(by='count_null', ascending=False ).head()<categorify>
df_test['raw_sentiment'] = df_test.apply(lambda x: find_sentiment(x['textID'], x['text'], x['sentiment']), axis=1)
 na_mask = df_test.raw_sentiment.isnull()
print(na_mask.sum() )
columns = ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu']
df[columns] = df[columns].fillna('nan')
 df_info = mount_df_info(df)
 df_info.sort_values(by='count_null', ascending=False ).head()<data_type_conversions>
test = df_test.copy()
tokenizer = AutoTokenizer.from_pretrained('.. /input/roberta-base/', do_lower_case=False)
 class Args:
post = True
tokenizer = tokenizer
offset = 4
batch_size = 4
workers = 0
args = Args()
class Args11:
post = True
tokenizer = tokenizer
offset = 7
batch_size = 4
workers = 0
args11 = Args11()
df.loc[df[df['GarageYrBlt'] == 2207].index, 'GarageYrBlt'] = 2010
df['GarageYrBlt'] = df['GarageYrBlt'].fillna(0 ).astype(int)
 columns = ['GarageCond', 'GarageFinish', 'GarageQual', 'GarageType']
df[columns] = df[columns].fillna('nan')
 df_info = mount_df_info(df)
 df_info.sort_values(by='count_null', ascending=False ).head()<categorify>
class TweetModel(nn.Module):
def __init__(self, pretrain_path=None, dropout=0.2, config=None):
super(TweetModel, self ).__init__()
if config is not None:
self.bert = AutoModel.from_config(config)
 else:
config = AutoConfig.from_pretrained(pretrain_path, output_hidden_states=True)
 self.bert = AutoModel.from_pretrained(
 pretrain_path, cache_dir=None, config=config)
 self.cnn = nn.Conv1d(self.bert.config.hidden_size*3, self.bert.config.hidden_size, 3, padding=1)
 self.gelu = nn.GELU()
self.whole_head = nn.Sequential(OrderedDict([ 
('dropout', nn.Dropout(0.1)) , 
('l1', nn.Linear(self.bert.config.hidden_size*3, 256)) , 
('act1', nn.GELU()), 
('dropout', nn.Dropout(0.1)) , 
('l2', nn.Linear(256, 2))
]))
self.se_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.inst_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.dropout = nn.Dropout(0.1)
 def forward(self, inputs, masks, token_type_ids=None, input_emb=None):
_, pooled_output, hs = self.bert(
 inputs, masks, token_type_ids=token_type_ids, inputs_embeds=input_emb)
 seq_output = torch.cat([hs[-1],hs[-2],hs[-3]], dim=-1)
 avg_output = torch.sum(seq_output*masks.unsqueeze(-1), dim=1, keepdim=False)
 avg_output = avg_output/torch.sum(masks, dim=-1, keepdim=True)
 whole_out = self.whole_head(avg_output)
 seq_output = self.gelu(self.cnn(seq_output.permute(0,2,1)).permute(0,2,1))
se_out = self.se_head(self.dropout(seq_output))
inst_out = self.inst_head(self.dropout(seq_output))
return whole_out, se_out[:, :, 0], se_out[:, :, 1], inst_out
columns = ['BsmtCond', 'BsmtExposure', 'BsmtQual', 'BsmtFinType2', 'BsmtFinType1']
df[columns] = df[columns].fillna('nan')
 df_info = mount_df_info(df)
 df_info.sort_values(by='count_null', ascending=False ).head()<define_variables>
def predict_wu(model: nn.Module, valid_df, valid_loader, args, progress=False)-> Dict[str, float]:
model.eval()
all_end_pred, all_whole_pred, all_start_pred, all_inst_out = [], [], [], []
if progress:
tq = tqdm.tqdm(total=len(valid_df))
with torch.no_grad() :
for tokens, types, masks, _, _, _, _, _, _, _ in valid_loader:
if progress:
batch_size = tokens.size(0)
 tq.update(batch_size)
 masks = masks.cuda()
tokens = tokens.cuda()
types = types.cuda()
whole_out, start_out, end_out, inst_out = model(tokens, masks, types)
 all_whole_pred.append(torch.softmax(whole_out, dim=-1)[:,1].cpu().numpy())
 inst_out = torch.softmax(inst_out, dim=-1)
 for idx in range(len(start_out)) :
length = torch.sum(masks[idx,:] ).item() -1
all_start_pred.append(torch.softmax(start_out[idx, args.offset:length], axis=-1 ).cpu())
 all_end_pred.append(torch.softmax(end_out[idx, args.offset:length], axis=-1 ).cpu())
 all_inst_out.append(inst_out[idx,:,1].cpu())
 assert all_start_pred[-1].dim() ==1
all_whole_pred = np.concatenate(all_whole_pred)
 if progress:
tq.close()
return all_whole_pred, all_start_pred, all_end_pred, all_inst_out
columns_disc = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig',
'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle',
'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'CentralAir', 'Electrical', 'Functional', 'GarageType', 'GarageFinish',
'PavedDrive', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition']
columns_disc_qual = ['ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'FireplaceQu',
'GarageQual', 'GarageCond', 'PoolQC']
columns_cont = ['LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']
columns_year = ['YearBuilt', 'GarageYrBlt', 'YearRemodAdd', 'YrSold', 'MoSold']
print('Columns disc', len(columns_disc))
print('Columns disc to cont', len(columns_disc_qual))
print('Columns cont', len(columns_cont))
print('Columns year', len(columns_year))<define_variables>
def _convrt_prob_partI(text, prob, max_char_len=160):
clean_text = " ".join(text.split())
 new_prob = []
p1, p2 = 0, 0
while p1 < len(text):
if text[p1] not in [" ", "\t", "\xa0"]:
if text[p1] == clean_text[p2]:
new_prob.append(prob[p1])
 p1 += 1
p2 += 1
else:
if p1 + 1 < len(text)and text[p1+1] not in [" ", "\t", "\xa0"]:
if clean_text[p2] == " ":
new_prob.append(prob[p1])
 p1 += 1
p2 += 1
else:
p1 += 1
else:
p1 += 1
if len(new_prob)< max_char_len:
new_prob = new_prob +(max_char_len - len(new_prob)) *[0]
return new_prob
columns_disc = columns_disc + columns_disc_qual<categorify>
def get_prediction_partI(weights_path, test, test_loader, args, output_name):
config = RobertaConfig.from_pretrained('.. /input/roberta-base', output_hidden_states=True)
 model = TweetModel(config=config)
 all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = [], [], [], []
for fold in range(10):
load_model(model, f'{weights_path}/best-model-%d.pt' % fold)
 model.cuda()
fold_whole_preds, fold_start_preds, fold_end_preds, fold_inst_preds = predict_wu(model, test, test_loader, args, progress=True)
 all_whole_preds.append(fold_whole_preds)
 all_start_preds.append(fold_start_preds)
 all_end_preds.append(fold_end_preds)
 all_inst_preds.append(fold_inst_preds)
 all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = ensemble(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test)
 word_preds, inst_word_preds, scores = get_predicts_from_token_logits(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test, args)
 start_char_prob, end_char_prob = get_char_prob(all_start_preds, all_end_preds, test, args)
 test['start_char_prob'] = start_char_prob
test['end_char_prob'] = end_char_prob
test['selected_text'] = word_preds
test['prob_start'] = test.apply(lambda x: _convrt_prob_partI(x['text'], x['start_char_prob']), axis=1)
 test['prob_end'] = test.apply(lambda x: _convrt_prob_partI(x['text'], x['end_char_prob']), axis=1)
 test.to_pickle(f'{output_name}.pkl')
 np.save(f"start_{output_name}.npy", np.array(test['prob_start'].tolist()))
np.save(f"end_{output_name}.npy", np.array(test['prob_end'].tolist()))
df_corr_disc = pd.DataFrame(columns=['feature', 'value', 'corr_with_sale_prices', 'count'])
 for _feature in columns_disc:
df_temp = pd.get_dummies(df_train[_feature])
 for _column in df_temp.columns:
_corr = stats.pointbiserialr(df_temp[_column], df_train['SalePrice'])
 df_corr_disc = df_corr_disc.append({'feature': _feature, 'value': _column, 'corr_with_sale_prices': _corr[0], 'count': sum(df_temp[_column])}, ignore_index=True)
 df_corr_disc.sort_values(by='corr_with_sale_prices' )<data_type_conversions>
collator = MyCollator()
test_set = TrainDataset(test, None, tokenizer=tokenizer, mode='test', offset=args.offset)
 test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, collate_fn=collator,
num_workers=args.workers)
 get_prediction_partI(weights_path='.. /input/roberta-v10-10',
test=test,
test_loader=test_loader,
args=args,
output_name='output_v10', 
)
 mem = psutil.virtual_memory()
print(f'{mem.percent:5} - {mem.free/1024**3:10.2f} - {mem.available/1024**3:10.2f} - {mem.used/1024**3:10.2f}' )
df_train['new_Fireplaces'] = df_train['Fireplaces'] * df_train['FireplaceQu'].map({'nan': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5} ).astype(int)
 df['new_Fireplaces'] = df['Fireplaces'] * df['FireplaceQu'].map({'nan': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5} ).astype(int)
 df_train[['new_Fireplaces', 'SalePrice']].corr(method='spearman' )<feature_engineering>
collator = MyCollator()
test_set = TrainDataset11(test, None, tokenizer=tokenizer, mode='test', offset=args11.offset)
 test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, collate_fn=collator,
num_workers=args.workers)
 get_prediction_partI(weights_path='.. /input/roberta-v11-10',
test=test,
test_loader=test_loader,
args=args11,
output_name='output_v11', 
)
 mem = psutil.virtual_memory()
print(f'{mem.percent:5} - {mem.free/1024**3:10.2f} - {mem.available/1024**3:10.2f} - {mem.used/1024**3:10.2f}' )
df_temp = df_train[df_train['YearRemodAdd'] > df_train['YearBuilt']]
plt.hist(df_temp['YearRemodAdd'] - df_temp['YearBuilt'], bins=30)
 plt.show()
df_train['new_has_remod'] = df_train['YearRemodAdd'] > df_train['YearBuilt']
df_train['new_time_remod'] = df_train['YearRemodAdd'] - df_train['YearBuilt']
df['new_has_remod'] = df['YearRemodAdd'] > df['YearBuilt']
df['new_time_remod'] = df['YearRemodAdd'] - df['YearBuilt']
print(stats.pointbiserialr(df_train['new_has_remod'], df_train['SalePrice']))
df_train[['new_time_remod', 'SalePrice']].corr(method='spearman' )<feature_engineering>
copyfile(src = ".. /input/tweet-sentiment/common.py", dst = ".. /working/common.py")
 copyfile(src = ".. /input/tweet-sentiment/dataset.py", dst = ".. /working/dataset.py")
 copyfile(src = ".. /input/tweet-sentiment/models.py", dst = ".. /working/models.py")
 copyfile(src = ".. /input/tweet-sentiment/metrics.py", dst = ".. /working/metrics.py")
 copyfile(src = ".. /input/tweet-sentiment/utils.py", dst = ".. /working/utils.py")
 copyfile(src = ".. /input/tweet-sentiment/predict_fn.py", dst = ".. /working/predict_fn.py")
 copyfile(src = ".. /input/tweet-sentiment/nlp_albumentations.py", dst = ".. /working/nlp_albumentations.py")
 copyfile(src = ".. /input/tweet-sentiment/transform.py", dst = ".. /working/transform.py")
 copyfile(src = ".. /input/tweet-sentiment/run_inference_kaggle.py", dst = ".. /working/run_inference_kaggle.py")
 set_seed(42)
df_train['new_garage_after_build'] = df_train['GarageYrBlt'] > df_train['YearBuilt']
df['new_garage_after_build'] = df['GarageYrBlt'] > df['YearBuilt']
print(stats.pointbiserialr(df_train['new_garage_after_build'], df_train['SalePrice']))<feature_engineering>
def get_model(model_name):
if model_name.startswith('roberta_base'):
model_info = {
'name': 'roberta-base',
'model_path': '.. /input/roberta-base' if ON_KAGGLE else 'roberta-base',
'from_pretrained': False if ON_KAGGLE else True,
'vocab_file': '.. /input/roberta-base/vocab.json',
'merges_file': '.. /input/roberta-base/merges.txt',
}
elif model_name.startswith('roberta_large'):
model_info = {
'name': 'roberta-large',
'model_path': '.. /input/roberta-large' if ON_KAGGLE else 'roberta-large',
'from_pretrained': False if ON_KAGGLE else True,
'vocab_file': '.. /input/roberta-large/vocab.json',
'merges_file': '.. /input/roberta-large/merges.txt',
}
else:
raise RuntimeError('%s is not implemented.' % model_name)
 model = TweetModel_v2(model_info)
 tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=model_info['vocab_file'],
merges_file=model_info['merges_file'],
lowercase=False,
add_prefix_space=True 
)
 return model, tokenizer
df_train['new_totalSF'] = df_train['TotalBsmtSF'] + df_train['1stFlrSF'] + df_train['2ndFlrSF'] + df_train['GarageArea']
df['new_totalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF'] + df['GarageArea']
df_train[['new_totalSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GarageArea', 'SalePrice']].corr(method='spearman' )<data_type_conversions>
N_FOLD = 10
POST_PROCESS = True
params = {
'models': [
'roberta_base_noRawSenti',
'roberta_large',
],
'batch_size': 4,
'workers': 1 if ON_KAGGLE else 8,
'max_len': 192,
'folds': list(x for x in range(N_FOLD)) ,
'limit': 0,
}
path_lib = {
'roberta_base': '.. /input/roberta-base/',
'roberta_base_noRawSenti': '.. /input/roberta-base-bs32-v2-0608',
'roberta_large': '.. /input/roberta-large-bs32-v2-0608',
}
df_train['BsmtFullBath'] = df_train['BsmtFullBath'].astype(int)
 df['BsmtFullBath'] = df['BsmtFullBath'].astype(int)
 df_train['BsmtHalfBath'] = df_train['BsmtHalfBath'].astype(int)
 df['BsmtHalfBath'] = df['BsmtHalfBath'].astype(int)
 df_train['new_others_room'] = df_train['TotRmsAbvGrd'] - df_train['BedroomAbvGr'] - df_train['KitchenAbvGr']
df['new_others_room'] = df['TotRmsAbvGrd'] - df['BedroomAbvGr'] - df['KitchenAbvGr']
df_train['new_all_room'] = df_train['TotRmsAbvGrd'] + df_train['BsmtFullBath'] + df_train['BsmtHalfBath'] + df_train['FullBath'] + df_train['HalfBath']
df['new_all_room'] = df['TotRmsAbvGrd'] + df['BsmtFullBath'] + df['BsmtHalfBath'] + df['FullBath'] + df['HalfBath']
df_train[['TotRmsAbvGrd', 'BedroomAbvGr', 'KitchenAbvGr', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'new_others_room', 'new_all_room', 'SalePrice']].corr(method='spearman' )<categorify>
def predict_II(model, data_loader, tokenizer):
start_probs, end_probs = [], []
with torch.no_grad() :
for i, d in enumerate(tqdm.tqdm(data_loader, ascii=True)) :
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
decode_selected = d["decode_selected"]
raw_tweets = d["raw_tweet"]
raw_selecteds = d["raw_selected_text"]
text_span = d["text_span"]
ids = ids.cuda()
token_type_ids = token_type_ids.cuda()
mask = mask.cuda()
start_idxs, end_idsx = [], []
outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 if len(outputs)== 2:
outputs_start, outputs_end = outputs
outputs_mask = None
elif len(outputs)== 3:
outputs_start, outputs_end, outputs_mask = outputs
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
for px, tweet in enumerate(orig_tweet):
raw_tweet = raw_tweets[px]
span_start, span_end = text_span[0][px], text_span[1][px]
span_start, span_end = int(span_start), int(span_end)
 token_ids = d["ids"][px][span_start: span_end]
clean_text = " ".join(raw_tweet.split())
 _, prob_char = convrt_prob_char_level(clean_text, token_ids, outputs_start[px, span_start: span_end], tokenizer)
 if len(prob_char)< 160:
prob_char += [0] *(160 - len(prob_char))
start_probs.append(prob_char)
 _, prob_char = convrt_prob_char_level(clean_text, token_ids, outputs_end[px, span_start: span_end], tokenizer)
 if len(prob_char)< 160:
prob_char += [0] *(160 - len(prob_char))
end_probs.append(prob_char)
 start_probs = np.array(start_probs)
 end_probs = np.array(end_probs)
 return start_probs, end_probs
_df_temp = pd.DataFrame()
for _columns in columns_disc_qual:
_df_temp[_columns] = df[_columns].map({'nan': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5} ).astype(int)
 df['new_overall'] = _df_temp[columns_disc_qual].sum(axis=1)/ len(columns_disc_qual )<define_variables>
for model_name in params['models']:
model, tokenizer = get_model(model_name)
 if model_name.endswith("noRawSenti"):
df_test_tmp = df_test.copy()
df_test_tmp['raw_sentiment'] = ""
tweet_dataset = TweetDataset(
 df=df_test_tmp,
sentiment_weights=[1,1,1],
tokenizer=tokenizer,
mode='test',
lower_case=0,
max_len=params['max_len'], 
)
 else:
tweet_dataset = TweetDataset(
 df=df_test,
sentiment_weights=[1,1,1],
tokenizer=tokenizer,
mode='test',
lower_case=0,
max_len=params['max_len'], 
)
 data_loader = DataLoader(
 tweet_dataset,
batch_size=params['batch_size'],
num_workers=0, 
)
 for i, fold in enumerate(params['folds']):
if model_name == "roberta_base_noRawSenti":
if fold < 5:
path = path_lib[model_name] + '-part2'
else:
path = path_lib[model_name]
elif model_name == "roberta_large":
if fold <= 2:
path = path_lib[model_name] + '-part1'
elif fold <= 5:
path = path_lib[model_name] + '-part2'
elif fold == 6:
path = path_lib[model_name] + '-part3-2'
elif fold == 7:
path = path_lib[model_name] + '-part3'
else:
path = path_lib[model_name] + '-part4'
else:
path = path_lib[model_name]
load_model(model, f"{path}/best_jac_{fold}.pt", multi2single=False)
 model.cuda()
model.eval()
probs_start_pred, probs_end_pred = predict_II(model, data_loader, tokenizer)
 if i == 0:
probs_start = probs_start_pred
probs_end = probs_end_pred
else:
probs_start += probs_start_pred
probs_end += probs_end_pred
probs_start /= len(params['folds'])
 probs_end /= len(params['folds'])
 df_test['prob_start'] = probs_start.tolist()
df_test['prob_end'] = probs_end.tolist()
df_test.to_pickle(f'{model_name}.pkl')
 np.save(f"start_{model_name}.npy", probs_start)
 np.save(f"end_{model_name}.npy", probs_end )
columns_disc = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig',
'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle',
'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'CentralAir', 'Electrical', 'Functional', 'GarageType', 'GarageFinish',
'PavedDrive', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition', 'MoSold',
'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'FireplaceQu',
'GarageQual', 'GarageCond', 'PoolQC',
'new_has_remod', 'new_garage_after_build']
columns_cont = ['LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
'new_Fireplaces', 'new_time_remod', 'new_time_sold', 'new_totalSF', 'new_others_room', 'new_all_room', 'new_overall']
columns_year = ['YearBuilt', 'GarageYrBlt', 'YearRemodAdd', 'YrSold']
print('Columns disc', len(columns_disc))
print('Columns cont', len(columns_cont))
print('Columns year', len(columns_year))<define_variables>
model_names = [
'roberta_base_noRawSenti',
'roberta_large',
'output_v10',
'output_v11',
]
for i, model_name in enumerate(model_names):
prob_start_tmp = np.load(f'start_{model_name}.npy')
 prob_end_tmp = np.load(f'end_{model_name}.npy')
 if i == 0:
prob_start = prob_start_tmp
prob_end = prob_end_tmp
else:
prob_start += prob_start_tmp
prob_end += prob_end_tmp
prob_start /= len(model_names)
 prob_end /= len(model_names)
 df_test['prob_start'] = prob_start.tolist()
df_test['prob_end'] = prob_end.tolist()
columns_cont = columns_cont + columns_year<categorify>
def _get_pred_char(df, probs_start, probs_end):
df['start_idx'] = np.argmax(probs_start, axis=1)
 df['end_idx'] = probs_end.shape[1] - np.argmax(probs_end[:, ::-1], axis=1)- 1
df['prob_start'] = probs_start.tolist()
df['prob_end'] = probs_end.tolist()
idxs = np.where(df.start_idx > df.end_idx)
 for idx in idxs[0]:
prob_start = df.prob_start.values[idx]
prob_end = df.prob_end.values[idx]
start_idx = df.start_idx.values[idx]
end_idx = df.end_idx.values[idx]
if prob_start[start_idx] > prob_end[end_idx] or end_idx == 0:
end_idx = len(prob_start)- np.argmax(prob_end[start_idx:][::-1])- 1
else:
start_idx = np.argmax(prob_start[:end_idx])
 df['start_idx'].iloc[idx] = start_idx
df['end_idx'].iloc[idx] = end_idx
df['pred_char'] = df.apply(lambda x: x['text_clean'][x['start_idx']: x['end_idx']+1], axis=1)
 return df
oneHot = OneHotEncoder(handle_unknown='ignore')
 df_onehot = oneHot.fit_transform(df[columns_disc] )<prepare_x_and_y>
def post_neutral(df):
df['select_pt'] = df.apply(lambda x: len(x['pred_char'].strip())/len(x['text_clean']), axis=1 ).values
raw_sents = ['neutral', 'sadness', 'worry', 'happiness', 'love', 'enthusiasm']
mm =(df['sentiment'] == 'neutral')&(df['raw_sentiment'].isin(raw_sents))
mm =(mm |\ 
(( df.select_pt > 0.85)&(df.sentiment.isin(['positive'])))|\ 
(( df.select_pt > 0.85)&(df.sentiment.isin(['negative'])))|\ 
(( df.select_pt < 0.2)&(df.sentiment.isin(['neutral']))))
df['pred_exp'] = df['pred_char'].values
df['pred_exp'].loc[mm] = df['text_clean'].loc[mm].values
print(f"
return df
def _post_shift_new(text, pred):
clean_text = " ".join(text.split())
 start_clean = clean_text.find(" ".join(pred.split()))
raw_pred = _convrt_back(text, pred, "neutral")
 raw_pred = raw_pred.strip()
start = text.find(raw_pred)
 end = start + len(raw_pred)
 extra_space = start - start_clean
if start>extra_space and extra_space>0:
if extra_space==1:
if text[start-1] in [',','.','?','!'] and text[start-2]!=' ':
start -= 1
elif extra_space==2:
start -= extra_space
if text[end-1] in [',','.','!','?','*']:
end -= 1
else:
end -=(extra_space-2)
 start -= extra_space
pred = text[start:end]
if pred.count("'")== 1:
if pred[0] == "'":
if text.find(pred)+ len(pred)< len(text)and text[text.find(pred)+ len(pred)] == "'":
pred += "'"
else:
if text.find(pred)- 1 >= 0 and text[text.find(pred)- 1] == "'":
pred = "'" + pred
return pred
def post_shift(df):
df['pred_final'] = df['pred_exp'].copy()
df['jac_text'] = df.apply(lambda x: jaccard(x['text'], x['pred_exp']), axis=1)
 mask =(df.sentiment != 'neutral')&(df.start_idx != 0)&(df.jac_text != 1)
 df['pred_final'].loc[mask] = df.apply(lambda x: _post_shift_new(x['text'], x['pred_exp']), axis=1 ).loc[mask].values
jac = df.apply(lambda x: jaccard(x['pred_exp'], x['pred_final']), axis=1)
 print(f"
return df
df_disc = pd.DataFrame.sparse.from_spmatrix(df_onehot, columns=oneHot.get_feature_names(df[columns_disc].columns)).astype(bool ).reset_index()
df_cont = df[columns_cont].reset_index()
df_processed = pd.concat([df_cont, df_disc], sort=False, axis=1 ).drop(columns=['index'] )<normalization>
POST_PROCESS=True
try:
df_test = _get_pred_char(df_test, np.array(df_test.prob_start.tolist()), np.array(df_test.prob_end.tolist()))
try:
if POST_PROCESS:
df_test = post_neutral(df_test)
 df_test = post_shift(df_test)
 df_test['selected_text'] = df_test['pred_final'].values
else:
df_test['selected_text'] = df_test['pred_char'].values
except:
df_test['selected_text'] = " "
except:
df_test = pd.merge(df_test[['textID']], df_pred1, how='left', on='textID' )
standard = StandardScaler()
df_processed[columns_cont] = standard.fit_transform(df_processed[columns_cont])
 transformer = FunctionTransformer(np.log1p)
 df_target = transformer.transform(df_target )<compute_test_metric>
df_test.to_csv("raw_prediction.csv", index=False)
 df_sub = df_test[['textID', 'selected_text']].copy()
df_sub.to_csv("submission.csv", index=False )