Kaggle competition: Santander Value Prediction Challenge
incorrect_labels_df = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 incorrect_labels_df = incorrect_labels_df[incorrect_labels_df['target'] > 1]
incorrect_texts = incorrect_labels_df.index.tolist()
train = train[~train.text.isin(incorrect_texts)]
train['keyword'].fillna('', inplace=True)
 train['final_text'] = train['keyword'] + ' ' + train['text']
test['keyword'].fillna('', inplace=True)
 test['final_text'] = test['keyword'] + ' ' + test['text']<load_pretrained>
from sklearn.model_selection import KFold
import lightgbm as lgb
from collections import Counter
import gc
text_values = train.final_text.values
labels = train.target.values
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )<string_transform>
def setup_dataset() :
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 subs = pd.read_csv(".. /input/sample_submission.csv")
 train["target"] = np.log1p(train["target"])
 subs["target"] = 0.0
train = prepare(train.replace(0, np.nan),"train")
 test = prepare(test.replace(0, np.nan),"test")
 return train,test,subs
def prepare(data,case):
if case=="train":
del_col=["ID","target"]
elif case=="test":
del_col=["ID"]
data_c = data.copy()
data['mean'] = data_c.drop(del_col,axis=1 ).mean(axis=1)
 data['std'] = data_c.drop(del_col,axis=1 ).std(axis=1)
 data['min'] = data_c.drop(del_col,axis=1 ).min(axis=1)
 data['max'] = data_c.drop(del_col,axis=1 ).max(axis=1)
 data['number_of_different'] = data_c.drop(del_col,axis=1 ).nunique(axis=1)
 data['non_zero_count'] = data_c.drop(del_col,axis=1 ).fillna(0 ).astype(bool ).sum(axis=1)
 d_matrix = data_c.drop(del_col,axis=1 ).fillna(0 ).apply(pair_check,axis=1)
 tmp_result = pd.DataFrame(list(d_matrix),columns=["not_pair_cnt", "not_pair_sum", "not_pair_max", "not_pair_min", "not_pair_mean","even_pair_cnt","even_pair_sum", "even_pair_max","even_pair_min","even_pair_mean"])
 for col in tmp_result.columns.tolist() :
data[col]=tmp_result[col]
return data
print('Original Text : ', text_values[1])
 print('Tokenized Text: ', tokenizer.tokenize(text_values[1]))
print('Token IDs : ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_values[1])) )<data_type_conversions>
def pair_check(column_values):
v = column_values.values.tolist()
not_pair_cnt=0
not_pair_sum=0
not_pair_max=0
not_pair_min=9999999999
even_pair_cnt=0
even_pair_sum=0
even_pair_max=0
even_pair_min=0
for key,cnt in Counter(v ).items() :
if key ==0:
continue
if cnt %2==1:
not_pair_cnt+=cnt
not_pair_sum+=key
if not_pair_max<key:
not_pair_max=key
if not_pair_min>key:
not_pair_min=key
else:
even_pair_cnt+=cnt
even_pair_sum+=key
if even_pair_max<key:
even_pair_max=key
if even_pair_min>key:
even_pair_min=key
if not_pair_cnt==0:
not_pair_mean=0
else:
not_pair_mean = not_pair_sum / not_pair_cnt
if even_pair_cnt==0:
even_pair_mean=0
else:
even_pair_mean = even_pair_sum / even_pair_cnt
return(not_pair_cnt, not_pair_sum, not_pair_max, not_pair_min, not_pair_mean, even_pair_cnt,even_pair_sum, even_pair_max,even_pair_min, even_pair_mean )
cls, sep, pad = '[CLS]', '[SEP]', '[PAD]'
print('Originally Format:', cls, sep, pad)
 print('Tokenized Text: ', tokenizer.tokenize(cls), tokenizer.tokenize(sep), tokenizer.tokenize(pad))
print('Token IDs : ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(cls)) , tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sep)) , tokenizer.convert_tokens_to_ids(tokenizer.tokenize(pad)) )<string_transform>
def train_lgb(d_train, d_test, d_subs, epoch, kfold_num=3,
delete_cols=[], lgb_params=lgb_params):
best_score=0
print("epoch:{}".format(epoch))
d_train = d_train.drop(delete_cols,axis=1)
 d_test = d_test.drop(delete_cols,axis=1)
 X_test = d_test.drop(["ID"],axis=1)
 fti_list=np.zeros(d_train.shape[1] - 2)
 low_fti=[]
del d_test
gc.collect()
for i,(train_idx, valid_idx)in enumerate(KFold(n_splits=kfold_num ).split(d_train)) :
X_train = d_train.iloc[train_idx,:].drop(["ID","target"],axis=1)
 X_valid = d_train.iloc[valid_idx,:].drop(["ID","target"],axis=1)
 Y_train = d_train.iloc[train_idx,:]["target"]
Y_valid = d_train.iloc[valid_idx,:]["target"]
lgb_train = lgb.Dataset(X_train, label=Y_train)
 lgb_valid = lgb.Dataset(X_valid, label=Y_valid)
 lgb_clf = lgb.train(
 lgb_params,
lgb_train,
num_boost_round=10000,
valid_sets=[lgb_train, lgb_valid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=50 
)
 fti_list += lgb_clf.feature_importance()
d_subs["target"] += np.expm1(lgb_clf.predict(X_test))
best_score += lgb_clf.best_score["valid"]['rmse']
d_subs["target"] /= kfold_num
best_score /= kfold_num
tee=[]
for i, feat in enumerate(X_train.columns.tolist()):
if fti_list[i]==0.0:
low_fti.append(feat)
 else:
tee.append([feat,fti_list[i]])
 print("best_score:{:.4f}".format(best_score))
return d_subs, low_fti, best_score,tee
print('Original Text : ', text_values[1])
 print('Tokenized Text : ', tokenizer.tokenize(text_values[1]))
print('Token IDs : ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_values[1])))
 print('Adding Special Tokens :', tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_values[1]))))
print('Adding Special Tokens Using Encode Func:', tokenizer.encode(text_values[1]))<categorify>
def execute_santander() :
print("start")
 train,d_test,subs = setup_dataset()
delete_cols=[]
fti_results=[]
score=99999.9
for epoch in range(10):
d_train = train.copy()
d_subs = subs.copy()
d_subs, low_fti, best_score,tee = train_lgb(d_train, d_test, d_subs, epoch=epoch,
kfold_num=3, delete_cols=delete_cols,
lgb_params=lgb_params)
 if score < best_score:
print("score is not improved")
 break
else:
score = best_score
delete_cols.extend(low_fti)
 print("delete_cols numbers:",len(delete_cols))
d_subs.to_csv("subs_lgb.csv",index=False)
 fti_results.append(tee)
 print("finish")
 return fti_results
def encode_fn(text_list):

 all_input_ids = []
for text in text_values:
input_ids = tokenizer.encode(
 text,
add_special_tokens = True,
max_length = 160,
pad_to_max_length = True,
return_tensors = 'pt' 
)
 all_input_ids.append(input_ids)
 all_input_ids = torch.cat(all_input_ids, dim=0)
 return all_input_ids<split>
fti_results = execute_santander()
epochs = 10
batch_size = 32
all_input_ids = encode_fn(text_values)
 labels = torch.tensor(labels)
 dataset = TensorDataset(all_input_ids, labels)
 train_size = int(0.95 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 train_dataloader = DataLoader(train_dataset, batch_size = batch_size, shuffle = True)
 val_dataloader = DataLoader(val_dataset, batch_size = batch_size, shuffle = False )<choose_model_class>
df = pd.DataFrame(fti_results[0],columns=["name","score"])
 df.sort_values(by="score",ascending=False )