Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<create_dataframe>
MY_CONFIG_PATH = '.. /input/base-pre-cv072469/base_pre_cv072469/base_pre_cv072469_config_20200612_150232.pkl'
MODEL1_FOLDER = ".. /input/large-cv0726033/RoBERTa-large-0726033 /"
MODEL1_CONFIG = "config_20200611_000411.pkl"
MODEL2_FOLDER = ".. /input/base-cv0722436/RoBERTa-base-0722436/"
MODEL2_CONFIG = "config_20200610_230921.pkl"
MODEL3_FOLDER = ".. /input/large-pre-cv072592/large_pre_cv072592/"
MODEL3_CONFIG = "large_pre_cv072592_config_20200612_112512.pkl"
MODEL4_FOLDER = ".. /input/base-pre-cv072469/base_pre_cv072469/"
MODEL4_CONFIG = "base_pre_cv072469_config_20200612_150232.pkl"
MODEL5_FOLDER = ".. /input/ooka-large-seed101/"
MODEL5_CONFIG = "config_20200614_233735.pkl"
MODEL6_FOLDER = ".. /input/ooka-base-seed101/ooka_base_seed101/"
MODEL6_CONFIG = "config_20200612_183046.pkl"
MODEL7_FOLDER = ".. /input/hiromu-large-seed101/"
MODEL7_CONFIG = "config_20200614_134559.pkl"
MODEL8_FOLDER = ".. /input/hiromu-base-seed101/hiromu_base_seed101/"
MODEL8_CONFIG = "config_20200614_124718.pkl"
ROBERTA_PATH = ".. /input/roberta-base"
REMODEL0 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold0.bin"
REMODEL1 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold1.bin"
REMODEL2 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold2.bin"
REMODEL3 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold3.bin"
REMODEL4 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold4.bin"
REMODEL5 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold0.bin"
REMODEL6 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold1.bin"
REMODEL7 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold2.bin"
REMODEL8 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold3.bin"
REMODEL9 = ".. /input/oof-bucketing-fold4/reranking_model_fold4.bin"
null_df =(df.isna().sum() / len(df)) *100
null_df = null_df.drop(null_df[null_df == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :null_df})
 missing_data.head(20 )<data_type_conversions>
!pip uninstall transformers -y
df['MSSubClass'] = df['MSSubClass'].apply(str)
 df['OverallCond'] = df['OverallCond'].astype(str)
 df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str )<categorify>
!pip uninstall tokenizers -y
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lb = LabelEncoder()
lb.fit(list(df[c].values))
df[c] = lb.transform(list(df[c].values))<feature_engineering>
!pip install --no-index --find-links /tmp/pip/cache-tokenizers/ tokenizers
df['TotalSF'] =(df['TotalBsmtSF']
+ df['1stFlrSF']
+ df['2ndFlrSF'])
 df['YrBltAndRemod'] = df['YearBuilt'] + df['YearRemodAdd']
df['Total_sqr_footage'] =(df['BsmtFinSF1']
+ df['BsmtFinSF2']
+ df['1stFlrSF']
+ df['2ndFlrSF'] 
)
 df['Total_Bathrooms'] =(df['FullBath']
+(0.5 * df['HalfBath'])
 + df['BsmtFullBath']
+(0.5 * df['BsmtHalfBath'])
)
 df['Total_porch_sf'] =(df['OpenPorchSF']
+ df['3SsnPorch']
+ df['EnclosedPorch']
+ df['ScreenPorch']
+ df['WoodDeckSF']
)<feature_engineering>
!pip install --no-index --find-links /tmp/pip/cache-transformers/ transformers
df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<compute_test_metric>
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath("__file__")))+"/input/tsepipeline/TSE-kaggle/input/src")
 pd.set_option("display.max_rows", 1000)
 pd.set_option("display.max_colwidth", 1000 )
stats.pearsonr(train[li_num[11]],target )<define_variables>
SENTIMENT_COMB = {
'pos':['positive'],
'neg':['negative'],
'neu':['neutral'],
'png':['positive','negative'],
'pnu':['neutral','positive'],
'nng':['neutral','negative'],
'pnn':['positive','negative','neutral'],
}
strong_num = ['OverallQual','YearBuilt','YearRemodAdd','TotalBsmtSF','1stFlrSF',
'FullBath','TotRmsAbvGrd','GarageYrBlt','GarageCars','GrLivArea']<define_variables>
import glob
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import BertTokenizer, get_cosine_schedule_with_warmup, get_linear_schedule_with_warmup
from utils.helper import seed_everything, read_pickle, Logger
from train.roberta_train import DEVICE
from train.roberta_train import TweetDataset, TweetCollate, RoBERTaBaseUncased
from train.roberta_train import calc_jaccard, create_valid, opt_th
from train.roberta_train import jaccard_apply
from train.roberta_train import valid, valid_total, ensemble, valid_02, valid_ensemble, create_test_candidates, create_test_candidates_ensemble
strong_obj = [ 'MSZoning', 'Neighborhood', 'Condition2', 'MasVnrType', 'ExterQual',
'BsmtQual','CentralAir', 'Electrical', 'KitchenQual', 'SaleType']<define_variables>
seed_everything(100 )
numeric_features = df.dtypes[df.dtypes != "object"].index<sort_values>
MY_CONFIG = read_pickle(MY_CONFIG_PATH )
skewness = df[numeric_features].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )<filter>
CONFIG1 = read_pickle(MODEL1_FOLDER+MODEL1_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL1_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath1 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG1 )
high_skewness = skewness[abs(skewness)> 0.5]
skew_feat = high_skewness.index<drop_column>
CONFIG2 = read_pickle(MODEL2_FOLDER+MODEL2_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL2_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath2 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG2 )
df[['MiscVal', 'PoolArea', 'haspool', 'LotArea', 'LowQualFinSF',
'3SsnPorch', 'LandSlope', 'KitchenAbvGr', 'BsmtFinSF2', 'EnclosedPorch',
'ScreenPorch', 'BsmtHalfBath', 'MasVnrArea', 'OpenPorchSF',
'WoodDeckSF', 'Total_porch_sf', '1stFlrSF', 'Total_sqr_footage',
'LotFrontage', 'GrLivArea', 'TotalSF', 'BsmtFinSF1', 'BsmtUnfSF',
'2ndFlrSF', 'TotRmsAbvGrd', 'Fireplaces', 'HalfBath', 'TotalBsmtSF',
'BsmtFullBath', 'OverallCond', 'YearBuilt', 'GarageFinish', 'LotShape',
'MoSold', 'Alley', 'BsmtExposure', 'KitchenQual', 'ExterQual', 'Fence',
'ExterCond', 'BsmtCond', 'PavedDrive', 'BsmtFinType2', 'GarageQual',
'CentralAir', 'GarageCond', 'GarageYrBlt', 'hasgarage', 'Functional',
'hasbsmt', 'Street', 'PoolQC']].head(3 )<feature_engineering>
CONFIG3 = read_pickle(MODEL3_FOLDER+MODEL3_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL3_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[7]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[6])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath3 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG3 )
lam = 0.15
for feat in skew_feat:
df[feat] = boxcox1p(df[feat], lam )<drop_column>
CONFIG4 = read_pickle(MODEL4_FOLDER+MODEL4_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL4_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[7]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[6])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath4 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG4 )
df[['MiscVal', 'PoolArea', 'haspool', 'LotArea', 'LowQualFinSF',
'3SsnPorch', 'LandSlope', 'KitchenAbvGr', 'BsmtFinSF2', 'EnclosedPorch',
'ScreenPorch', 'BsmtHalfBath', 'MasVnrArea', 'OpenPorchSF',
'WoodDeckSF', 'Total_porch_sf', '1stFlrSF', 'Total_sqr_footage',
'LotFrontage', 'GrLivArea', 'TotalSF', 'BsmtFinSF1', 'BsmtUnfSF',
'2ndFlrSF', 'TotRmsAbvGrd', 'Fireplaces', 'HalfBath', 'TotalBsmtSF',
'BsmtFullBath', 'OverallCond', 'YearBuilt', 'GarageFinish', 'LotShape',
'MoSold', 'Alley', 'BsmtExposure', 'KitchenQual', 'ExterQual', 'Fence',
'ExterCond', 'BsmtCond', 'PavedDrive', 'BsmtFinType2', 'GarageQual',
'CentralAir', 'GarageCond', 'GarageYrBlt', 'hasgarage', 'Functional',
'hasbsmt', 'Street', 'PoolQC']].head(3 )<categorify>
CONFIG5 = read_pickle(MODEL5_FOLDER+MODEL5_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL5_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath5 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG5 )
df = pd.get_dummies(df)
 print(df.shape )<split>
CONFIG6 = read_pickle(MODEL6_FOLDER+MODEL6_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL6_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath6 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG6 )
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]<concatenate>
CONFIG7 = read_pickle(MODEL7_FOLDER+MODEL7_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL7_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath7 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG7 )
new_train = pd.concat([new_train,target], axis=1, sort=False )<sort_values>
CONFIG8 = read_pickle(MODEL8_FOLDER+MODEL8_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL8_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath8 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG8 )
col_corr_dict = corr_new_train['SalePrice'].sort_values(ascending=False ).to_dict()<find_best_params>
for item in set(MY_CONFIG.keys())- set(CONFIG1.keys()):
CONFIG1[item] = MY_CONFIG[item]
CONFIG2[item] = MY_CONFIG[item]
best_columns=[]
for key,value in col_corr_dict.items() :
if(( value>=0.33)&(value<0.9)) |(value<=-0.325):
best_columns.append(key)
 print(len(best_columns))<drop_column>
JAC_TH = 0.46
_, test_df = preprocess_df(ORI_DATA_DIR, use_sentiment=SENTIMENT_COMB[CONFIG1['USE_SENTIMENT']], show_head=False)
 pred_ensemble = create_test_candidates_ensemble(test_df, filepath1, filepath2, filepath3, filepath4, CONFIG1, CONFIG2, CONFIG3, CONFIG4, JAC_TH=JAC_TH, ratio=[0.4, 0.1, 0.3, 0.2, 0.4, 0.1, 0.3, 0.2], en_ratio=[0.5, 0.5], num=2 )
new_train = new_train.drop(['SalePrice'], axis=1)
 new_train = new_train.drop(['Id'], axis=1)
 new_test = new_test.drop(['Id'], axis=1 )<import_modules>
seed_everything(100)
 MAX_LEN = 256
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
 cand_nums = 5
val_nums = 5
from sklearn.model_selection import KFold, cross_val_score, GridSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.linear_model import ElasticNet, Lasso, LinearRegression<import_modules>
class RerankingDataset:
def __init__(self, tweet, sentiment, selected_text, jaccard, score):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
self.jaccard = jaccard
self.score = score
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
tweet = self.tweet[item]
selected_text = self.selected_text[item]
sentiment = self.sentiment[item]
jaccard = self.jaccard[item]
score = self.score[item]
new_tweet = str(tweet)+ " 
 
 " + str(selected_text)
 tok_tweet = TOKENIZER.encode(new_tweet)
 input_ids_orig = tok_tweet.ids
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 return {
'orig_tweet' : tweet,
'sentiment' : sentiment,
'orig_selected' : selected_text,
'jaccard' : jaccard,
'score' : score,
'ids' : input_ids,
'mask' : mask,
'token_type_ids' : token_type_ids
}
class RerankingCollate:
def __init__(self):
self.CONFIG = {}
self.CONFIG['BUCKET'] = True
self.CONFIG['MAX_LEN'] = MAX_LEN
def __call__(self, batch):
out = {
'orig_tweet' : [],
'sentiment' : [],
'orig_selected' : [],
'jaccard' : [],
'score' : [],
'ids' : [],
'mask' : [],
'token_type_ids' : [],
}
for i in range(len(batch)) :
for k, v in batch[i].items() :
out[k].append(v)
 if self.CONFIG['BUCKET']:
max_pad = 0
for p in out['ids']:
if len(p)>max_pad:
max_pad = len(p)
 else:
max_pad = self.CONFIG['MAX_LEN']
for i in range(len(batch)) :
tokenized_text = out['ids'][i]
token_type_ids = out['token_type_ids'][i]
mask = out['mask'][i]
text_len = len(tokenized_text)
 out['ids'][i] =(tokenized_text + [1] *(max_pad - text_len)) [:max_pad]
out['token_type_ids'][i] =(token_type_ids + [0] *(max_pad - text_len)) [:max_pad]
out['mask'][i] =(mask + [0] *(max_pad - text_len)) [:max_pad]
out['jaccard'] = torch.tensor(out['jaccard'], dtype=torch.float)
 out['score'] = torch.tensor(out['score'], dtype=torch.float)
 out['ids'] = torch.tensor(out['ids'], dtype=torch.long)
 out['mask'] = torch.tensor(out['mask'], dtype=torch.long)
 out['token_type_ids'] = torch.tensor(out['token_type_ids'], dtype=torch.long)
 return out
from sklearn.preprocessing import RobustScaler<normalization>
class myModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(myModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=conf)
 self.dropout = nn.Dropout(0.1)
 self.out_proj = nn.Linear(768*2, 1)
 def forward(self, ids, mask, token_type_ids):
o1, o2, _ = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 apool = torch.mean(o1, 1)
 mpool, _ = torch.max(o1, 1)
 cat = torch.cat(( apool, mpool), 1)
 bo = self.dropout(cat)
 p2 = self.out_proj(bo)
 return p2
rbst_scaler=RobustScaler()
X_rbst=rbst_scaler.fit_transform(new_train)
 test_rbst=rbst_scaler.transform(new_test )<import_modules>
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
model0 = myModel(conf=model_config)
 model0.to(device)
 model0.load_state_dict(torch.load(REMODEL0))
model1 = myModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(REMODEL1))
model2 = myModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load(REMODEL2))
model3 = myModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load(REMODEL3))
model4 = myModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load(REMODEL4))
model5 = myModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load(REMODEL5))
model6 = myModel(conf=model_config)
 model6.to(device)
 model6.load_state_dict(torch.load(REMODEL6))
model7 = myModel(conf=model_config)
 model7.to(device)
 model7.load_state_dict(torch.load(REMODEL7))
model8 = myModel(conf=model_config)
 model8.to(device)
 model8.load_state_dict(torch.load(REMODEL8))
model9 = myModel(conf=model_config)
 model9.to(device)
 model9.load_state_dict(torch.load(REMODEL9))
print('model load' )
import statsmodels.api as sm<compute_test_metric>
dev = pd.read_csv('/kaggle/working/test_candidate.csv')
 dev
model = sm.OLS(target.values, new_train )<train_model>
valid_dataset = RerankingDataset(
 tweet=dev.tweet.values,
sentiment=dev.sentiment.values,
selected_text=dev.candidate.values,
jaccard=dev.jaccard.values,
score=dev.score.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=val_nums,
collate_fn=RerankingCollate() ,
num_workers=0,
)
re = model.fit()<create_dataframe>
def eval(valid_data_loader):
model0.eval()
model1.eval()
model2.eval()
model3.eval()
model4.eval()
model5.eval()
model6.eval()
model7.eval()
model8.eval()
model9.eval()
all_jaccard = 0
all_num = 0
rets = []
with torch.no_grad() :
for bi, d in enumerate(valid_data_loader):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
jaccard = d["jaccard"]
score = d["score"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 logits0 = model0(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits6 = model6(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits7 = model7(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits8 = model8(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 logits9 = model9(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 maxind = 0
maxjac =(logits0[0][0] + logits1[0][0] + logits2[0][0] + logits3[0][0] + logits4[0][0] + logits5[0][0] + logits6[0][0] + logits7[0][0] + logits8[0][0] + logits9[0][0])/ 10 + score[0] * 0.5
for i in range(1, val_nums):
tmpjac =(logits0[i][0] + logits1[i][0] + logits2[i][0] + logits3[i][0] + logits4[i][0] + logits5[i][0] + logits6[i][0] + logits7[i][0] + logits8[i][0] + logits9[i][0])/ 10 + score[i] * 0.5
if maxjac < tmpjac:
maxind = i
maxjac = tmpjac
rets.append(orig_selected[maxind])
 return rets
vif = pd.DataFrame()
vif['Features'] = new_train.columns
vif['vif'] = [variance_inflation_factor(
 new_train.values, i)for i in range(new_train.shape[1])]<sort_values>
cand = eval(valid_data_loader )
vif.sort_values(by='vif',ascending=False)[165:190]<normalization>
def pp(filtered_output, real_tweet):
filtered_output = ' '.join(filtered_output.split())
 if len(real_tweet.split())< 2:
filtered_output = real_tweet
else:
if len(filtered_output.split())== 1:
if filtered_output.endswith(".. "):
if real_tweet.startswith(" "):
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\.) \1{2,}', '', filtered_output)
 else:
filtered_output = re.sub(r'(\.) \1{2,}', '.', filtered_output)
 else:
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\.) \1{2,}', '.', filtered_output)
 else:
filtered_output = re.sub(r'(\.) \1{2,}', '.. ', filtered_output)
 return filtered_output
if filtered_output.endswith('!!'):
if real_tweet.startswith(" "):
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\!)\1{2,}', '', filtered_output)
 else:
filtered_output = re.sub(r'(\!)\1{2,}', '!', filtered_output)
 else:
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\!)\1{2,}', '!', filtered_output)
 else:
filtered_output = re.sub(r'(\!)\1{2,}', '!!', filtered_output)
 return filtered_output
if real_tweet.startswith(" "):
filtered_output = filtered_output.strip()
text_annotetor = ' '.join(real_tweet.split())
 start = text_annotetor.find(filtered_output)
 end = start + len(filtered_output)
 start -= 0
end += 2
flag = real_tweet.find(" ")
 if flag < start:
filtered_output = real_tweet[start:end]
if " " in real_tweet and not real_tweet.startswith(" "):
filtered_output = filtered_output.strip()
text_annotetor = re.sub(" {2,}", " ", real_tweet)
 start = text_annotetor.find(filtered_output)
 end = start + len(filtered_output)
 start -= 0
end += 2
flag = real_tweet.find(" ")
 if flag < start:
filtered_output = real_tweet[start:end]
return filtered_output
rbst_scaler=RobustScaler()
X_rbst=rbst_scaler.fit_transform(new_train)
 test_rbst=rbst_scaler.transform(new_test )<import_modules>
final_output = []
test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 tweet = list(test.text.values)
 for c, t in zip(cand, tweet):
final_output.append(pp(c, t))
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold<choose_model_class>
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample
<choose_model_class><EOS> 
sample.to_csv('submission.csv', index=False )