Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<compute_train_metric>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
preds -= preds.min()
preds / preds.max()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]<merge>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
def ProjectOnMean(data1, data2, columnName):
grpOutcomes = data1.groupby(list([columnName])) ['target'].mean().reset_index()
grpCount = data1.groupby(list([columnName])) ['target'].count().reset_index()
grpOutcomes['cnt'] = grpCount.target
grpOutcomes.drop('cnt', inplace=True, axis=1)
 outcomes = data2['target'].values
x = pd.merge(data2[[columnName, 'target']], grpOutcomes,
suffixes=('x_', ''),
how='left',
on=list([columnName]),
left_index=True)['target']
return x.fillna(x.mean() ).values
def GetData(strdirectory):
highcardinality = ['ps_car_02_cat',
'ps_car_09_cat',
'ps_ind_04_cat',
'ps_ind_05_cat',
'ps_car_03_cat',
'ps_ind_08_bin',
'ps_car_05_cat',
'ps_car_08_cat',
'ps_ind_06_bin',
'ps_ind_07_bin',
'ps_ind_12_bin',
'ps_ind_18_bin',
'ps_ind_17_bin',
'ps_car_07_cat',
'ps_car_11_cat',
'ps_ind_09_bin',
'ps_car_10_cat',
'ps_car_04_cat',
'ps_car_01_cat',
'ps_ind_02_cat',
'ps_ind_10_bin',
'ps_ind_11_bin',
'ps_car_06_cat',
'ps_ind_13_bin',
'ps_ind_16_bin']
train = pd.read_csv(strdirectory+'train.csv')
 test = pd.read_csv(strdirectory+'test.csv')
 train['missing'] =(train==-1 ).sum(axis=1 ).astype(float)
 test['missing'] =(test==-1 ).sum(axis=1 ).astype(float)
 unwanted = train.columns[train.columns.str.startswith('ps_calc_')]
train.drop(unwanted,inplace=True,axis=1)
 test.drop(unwanted,inplace=True,axis=1)
 test['target'] = np.nan
feats = list(set(train.columns ).difference(set(['id','target'])))
 feats = list(['id'])+feats +list(['target'])
 train = train[feats]
test = test[feats]
blindloodata = None
folds = 5
kf = StratifiedKFold(n_splits=folds,shuffle=True,random_state=2017)
 for i,(train_index, test_index)in enumerate(kf.split(range(train.shape[0]),train.target)) :
print('Fold:',i)
 blindtrain = train.iloc[test_index].copy()
vistrain = train.iloc[train_index].copy()
for c in highcardinality:
blindtrain.insert(1,'loo_'+c, ProjectOnMean(vistrain,
blindtrain,c))
if(blindloodata is None):
blindloodata = blindtrain.copy()
else:
blindloodata = pd.concat([blindloodata,blindtrain])
 for c in highcardinality:
test.insert(1,'loo_'+c, ProjectOnMean(train,
test,c))
train = blindloodata
print('Scale values')
 ss = StandardScaler()
features = train.columns[1:-1]
ss.fit(pd.concat([train[features],test[features]]))
train[features] = ss.transform(train[features])
 test[features] = ss.transform(test[features])
 train[features] = np.round(train[features], 6)
 test[features] = np.round(test[features], 6)
 return highcardinality, train, test<split>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
strdirectory = '.. /input/'
highcardinality, train, test = GetData(strdirectory )<feature_engineering>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def GP1(data):
v = pd.DataFrame()
v["i1"] = 0.020000*np.tanh(((( data["ps_car_12"] +(( data["ps_car_15"] + data["ps_reg_01"])+ data["loo_ps_ind_16_bin"])) * 2.0)* 2.0))
v["i2"] = 0.019910*np.tanh(((( data["loo_ps_car_04_cat"] +(data["loo_ps_ind_05_cat"] +(data["loo_ps_car_06_cat"] + data["loo_ps_ind_17_bin"])))* 2.0)* 2.0))
v["i3"] = 0.020000*np.tanh(((( data["loo_ps_car_05_cat"] + data["loo_ps_ind_16_bin"])+(data["loo_ps_car_09_cat"] +(data["loo_ps_car_11_cat"] * 2.0)))* 2.0))
v["i4"] = 0.020000*np.tanh(( data["ps_car_15"] +(((data["ps_reg_02"] + data["loo_ps_ind_06_bin"])+ data["ps_reg_03"])+ data["loo_ps_car_11_cat"])))
 v["i5"] = 0.020000*np.tanh(((((data["loo_ps_ind_05_cat"] + data["loo_ps_ind_17_bin"])+ data["loo_ps_car_02_cat"])+ data["loo_ps_car_04_cat"])* 2.0))
v["i6"] = 0.020000*np.tanh(((data["loo_ps_ind_17_bin"] + data["loo_ps_ind_05_cat"])+(data["ps_car_13"] +(data["ps_car_13"] + data["ps_reg_02"]))))
v["i7"] = 0.020000*np.tanh(((data["loo_ps_ind_16_bin"] +(( data["loo_ps_car_03_cat"] + data["ps_car_13"])* 2.0)) * 2.0))
v["i8"] = 0.019945*np.tanh(((data["loo_ps_car_03_cat"] + data["loo_ps_car_04_cat"])+(( data["loo_ps_car_06_cat"] + data["loo_ps_ind_17_bin"])+ data["loo_ps_car_01_cat"])))
 v["i9"] = 0.020000*np.tanh(((( data["ps_car_13"] + data["loo_ps_car_07_cat"])* 2.0)+(data["loo_ps_ind_06_bin"] + data["loo_ps_ind_16_bin"])))
 v["i10"] = 0.020000*np.tanh(((( data["loo_ps_car_09_cat"] +(data["loo_ps_ind_09_bin"] +(data["loo_ps_ind_06_bin"] + data["loo_ps_car_07_cat"])))* 2.0)* 2.0))
v["i11"] = 0.020000*np.tanh(((data["loo_ps_car_11_cat"] +(( data["loo_ps_ind_05_cat"] + data["loo_ps_ind_06_bin"])+ data["loo_ps_car_07_cat"])) * 2.0))
v["i12"] = 0.020000*np.tanh(((((( data["ps_car_13"] * 2.0)+ data["loo_ps_car_03_cat"])/2.0)+(data["ps_reg_01"] + data["loo_ps_ind_05_cat"])) * 2.0))
v["i13"] = 0.020000*np.tanh(((data["loo_ps_ind_16_bin"] +(( data["ps_car_13"] +(data["loo_ps_car_01_cat"] + data["loo_ps_car_07_cat"])) * 2.0)) * 2.0))
v["i14"] = 0.020000*np.tanh(((((( data["loo_ps_car_02_cat"] + data["loo_ps_ind_07_bin"])+ data["loo_ps_ind_05_cat"])+ data["loo_ps_ind_07_bin"])* 2.0)* 2.0))
v["i15"] = 0.020000*np.tanh(( data["ps_car_13"] +(data["loo_ps_ind_05_cat"] +(( data["loo_ps_ind_06_bin"] + data["loo_ps_car_01_cat"])+ data["loo_ps_car_09_cat"]))))
v["i16"] = 0.020000*np.tanh(((((data["loo_ps_ind_06_bin"] + data["ps_reg_02"])+ data["ps_car_13"])+ data["loo_ps_car_01_cat"])+ data["ps_car_13"]))
v["i17"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] + data["loo_ps_ind_17_bin"])+(data["ps_car_12"] + data["ps_reg_01"])) - data["ps_ind_15"]))
v["i18"] = 0.020000*np.tanh(((( data["ps_car_15"] +(( data["ps_reg_03"] + data["loo_ps_car_09_cat"])* 2.0)) + data["ps_reg_02"])* 2.0))
v["i19"] = 0.020000*np.tanh(( data["ps_car_13"] +(( data["ps_reg_03"] + data["loo_ps_car_07_cat"])-(data["ps_ind_15"] - data["loo_ps_car_03_cat"]))))
v["i20"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] +(data["loo_ps_car_07_cat"] +(data["loo_ps_car_03_cat"] + data["loo_ps_car_09_cat"])))* 2.0)* 2.0))
v["i21"] = 0.020000*np.tanh(((( data["loo_ps_car_09_cat"] +(data["loo_ps_ind_06_bin"] - data["ps_ind_15"])) + data["ps_reg_02"])* 2.0))
v["i22"] = 0.020000*np.tanh(((((((data["loo_ps_ind_17_bin"] + data["ps_reg_03"])* 2.0)* 2.0)* 2.0)+ data["loo_ps_car_11_cat"])* 2.0))
v["i23"] = 0.020000*np.tanh(((data["loo_ps_car_01_cat"] + data["loo_ps_ind_07_bin"])+(data["loo_ps_ind_05_cat"] +(data["ps_reg_03"] + data["loo_ps_ind_09_bin"]))))
v["i24"] = 0.020000*np.tanh(((((data["loo_ps_ind_05_cat"] - data["ps_ind_15"])+ data["ps_ind_03"])+ data["loo_ps_car_07_cat"])+ data["loo_ps_ind_05_cat"]))
v["i25"] = 0.020000*np.tanh(((data["ps_reg_02"] -(( data["loo_ps_ind_02_cat"] * 2.0)* data["ps_ind_03"])) + data["ps_car_13"]))
v["i26"] = 0.020000*np.tanh(((( data["ps_car_13"] + data["loo_ps_car_09_cat"])+(data["loo_ps_car_07_cat"] + data["ps_ind_03"])) - data["ps_ind_15"]))
v["i27"] = 0.020000*np.tanh(((data["loo_ps_car_07_cat"] + data["loo_ps_ind_06_bin"])+(data["loo_ps_car_01_cat"] +(data["loo_ps_ind_16_bin"] - data["ps_ind_15"]))))
v["i28"] = 0.020000*np.tanh(((((data["ps_reg_03"] +(data["ps_reg_03"] * 2.0)) + data["loo_ps_ind_04_cat"])+ data["ps_reg_02"])* 2.0))
v["i29"] = 0.020000*np.tanh(((data["loo_ps_ind_09_bin"] +(data["loo_ps_ind_17_bin"] +(data["ps_car_13"] + data["ps_ind_01"])))+ data["loo_ps_car_09_cat"]))
v["i30"] = 0.020000*np.tanh(( data["loo_ps_car_07_cat"] +(((data["loo_ps_ind_02_cat"] - data["ps_ind_15"])+ data["loo_ps_ind_09_bin"])+ data["ps_reg_01"])))
 v["i31"] = 0.020000*np.tanh(((data["loo_ps_ind_02_cat"] +(( data["loo_ps_car_01_cat"] + data["loo_ps_ind_08_bin"])+ data["loo_ps_ind_02_cat"])) + data["loo_ps_ind_07_bin"]))
v["i32"] = 0.020000*np.tanh(((((( data["ps_ind_03"] +(data["ps_ind_03"] * data["ps_ind_03"])) * 2.0)* 2.0)* 2.0)* 2.0))
v["i33"] = 0.020000*np.tanh(((( data["loo_ps_car_01_cat"] +(data["loo_ps_car_01_cat"] - data["ps_ind_15"])) - data["ps_ind_15"])+ data["loo_ps_car_07_cat"]))
v["i34"] = 0.020000*np.tanh(((( data["loo_ps_car_07_cat"] +(data["loo_ps_ind_16_bin"] - data["loo_ps_ind_18_bin"])) * 2.0)* 2.0))
v["i35"] = 0.020000*np.tanh(((((( data["loo_ps_ind_05_cat"] * 2.0)+(data["loo_ps_ind_07_bin"] * data["ps_ind_03"])) * 2.0)* 2.0)* 2.0))
v["i36"] = 0.020000*np.tanh(((((data["ps_ind_03"] * data["ps_ind_03"])+ data["ps_ind_03"])+(data["loo_ps_ind_05_cat"] * 2.0)) * 2.0))
v["i37"] = 0.020000*np.tanh(((((( data["loo_ps_ind_17_bin"] * 2.0)+(data["loo_ps_ind_05_cat"] + data["loo_ps_car_11_cat"])) * 2.0)* 2.0)* 2.0))
v["i38"] = 0.020000*np.tanh(((data["ps_car_13"] +(data["loo_ps_car_08_cat"] +(data["ps_ind_03"] * data["ps_ind_03"])))+ data["loo_ps_car_09_cat"]))
v["i39"] = 0.019996*np.tanh(((((( data["ps_car_15"] + data["loo_ps_car_07_cat"])+(data["ps_reg_03"] * 2.0)) * 2.0)* 2.0)* 2.0))
v["i40"] = 0.020000*np.tanh(((((data["loo_ps_ind_17_bin"] + data["loo_ps_ind_02_cat"])* 2.0)* 2.0)+ data["ps_car_15"]))
v["i41"] = 0.020000*np.tanh(( 1.0 +(( data["loo_ps_ind_04_cat"] - data["ps_ind_15"])+(data["loo_ps_car_03_cat"] - data["ps_ind_15"]))))
v["i42"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] +(data["loo_ps_ind_05_cat"] * 2.0)) +(data["ps_ind_03"] * data["ps_ind_03"])) * 2.0))
v["i43"] = 0.020000*np.tanh(((data["loo_ps_ind_06_bin"] +(((data["loo_ps_ind_06_bin"] + data["loo_ps_ind_09_bin"])* data["ps_ind_03"])* 2.0)) * 2.0))
v["i44"] = 0.020000*np.tanh(((((data["loo_ps_car_01_cat"] + data["loo_ps_ind_09_bin"])+(data["ps_ind_01"] + data["loo_ps_ind_09_bin"])) * 2.0)* 2.0))
v["i45"] = 0.020000*np.tanh(((( data["loo_ps_car_07_cat"] + data["loo_ps_ind_05_cat"])+(( data["loo_ps_car_07_cat"] + data["loo_ps_ind_05_cat"])* 2.0)) * 2.0))
v["i46"] = 0.020000*np.tanh(((( data["loo_ps_ind_04_cat"] +(data["loo_ps_ind_04_cat"] + data["ps_ind_01"])) + data["loo_ps_car_09_cat"])+ data["ps_ind_03"]))
v["i47"] = 0.020000*np.tanh(( data["ps_ind_01"] +(( data["loo_ps_car_05_cat"] *(data["ps_ind_01"] * 2.0)) - data["ps_ind_15"])))
 v["i48"] = 0.020000*np.tanh(( data["loo_ps_ind_17_bin"] +(( data["loo_ps_ind_16_bin"] * data["ps_car_12"])+ data["loo_ps_ind_05_cat"])))
 v["i49"] = 0.020000*np.tanh(((8.70196723937988281)*(( 8.70196723937988281)*(data["ps_ind_03"] *(( 0.0)- data["loo_ps_ind_02_cat"])))))
 v["i50"] = 0.020000*np.tanh(( data["ps_car_15"] +(( data["loo_ps_ind_17_bin"] * data["loo_ps_ind_17_bin"])-(data["loo_ps_car_04_cat"] * data["ps_car_11"]))))
v["i51"] = 0.020000*np.tanh(((data["loo_ps_ind_17_bin"] - data["ps_reg_01"])*(( 7.16651058197021484)* data["loo_ps_car_01_cat"])))
 v["i52"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] - data["ps_car_11"])- data["ps_ind_15"])+(data["loo_ps_ind_05_cat"] - data["ps_ind_15"])))
 v["i53"] = 0.020000*np.tanh(((data["loo_ps_car_09_cat"] + data["loo_ps_ind_05_cat"])+(data["loo_ps_ind_06_bin"] * data["loo_ps_ind_05_cat"])))
 v["i54"] = 0.020000*np.tanh(((data["loo_ps_car_06_cat"] +(data["loo_ps_ind_17_bin"] +(data["ps_ind_01"] + data["missing"])))* data["loo_ps_car_05_cat"]))
v["i55"] = 0.020000*np.tanh(((data["ps_reg_03"] +(( data["ps_reg_03"] + data["loo_ps_ind_04_cat"])* 2.0)) * 2.0))
v["i56"] = 0.020000*np.tanh(((data["loo_ps_car_01_cat"] +(data["loo_ps_car_01_cat"] - data["loo_ps_car_02_cat"])) * 2.0))
v["i57"] = 0.020000*np.tanh(( -(( data["ps_ind_03"] *(((( data["loo_ps_ind_02_cat"] * 2.0)* 2.0)* 2.0)* 2.0)))))
 v["i58"] = 0.019996*np.tanh(( data["loo_ps_car_07_cat"] +(data["ps_ind_03"] *(data["ps_ind_03"] +(data["loo_ps_ind_17_bin"] + data["loo_ps_car_11_cat"])))))
 v["i59"] = 0.020000*np.tanh(((( -2.0 + data["loo_ps_ind_05_cat"])- data["ps_car_11"])* 2.0))
v["i60"] = 0.020000*np.tanh(( data["loo_ps_ind_09_bin"] * data["loo_ps_car_03_cat"]))
v["i61"] = 0.020000*np.tanh(((((( data["loo_ps_ind_04_cat"] +(data["loo_ps_ind_08_bin"] * data["ps_ind_03"])) * 2.0)* 2.0)* 2.0)* 2.0))
v["i62"] = 0.020000*np.tanh(( data["ps_car_15"] +(data["loo_ps_ind_17_bin"] * data["loo_ps_car_03_cat"])))
 v["i63"] = 0.020000*np.tanh(((data["loo_ps_ind_06_bin"] *(( data["loo_ps_ind_05_cat"] + data["loo_ps_ind_05_cat"])- data["loo_ps_car_11_cat"])) * 2.0))
v["i64"] = 0.019992*np.tanh(( data["loo_ps_car_05_cat"] *(data["ps_car_12"] +(data["loo_ps_ind_17_bin"] + data["loo_ps_car_08_cat"]))))
v["i65"] = 0.020000*np.tanh(((((data["missing"] * 2.0)* data["loo_ps_ind_02_cat"])* 2.0)+ data["loo_ps_ind_09_bin"]))
v["i66"] = 0.020000*np.tanh(((((( data["missing"] / 2.0)+(data["loo_ps_car_08_cat"] * 2.0)) * 2.0)* 2.0)* data["loo_ps_ind_02_cat"]))
v["i67"] = 0.020000*np.tanh(((-(((data["ps_reg_02"] * data["loo_ps_car_06_cat"])+(data["ps_reg_03"] * data["ps_ind_01"])))) * 2.0))
v["i68"] = 0.020000*np.tanh(((((data["ps_ind_01"] + data["loo_ps_car_09_cat"])+(( data["loo_ps_car_11_cat"] + data["loo_ps_car_09_cat"])/2.0)) /2.0)* data["loo_ps_ind_09_bin"]))
v["i69"] = 0.020000*np.tanh(( data["loo_ps_ind_06_bin"] *(((data["loo_ps_ind_06_bin"] - data["ps_reg_01"])- data["ps_car_15"])+ data["loo_ps_ind_05_cat"])))
 v["i70"] = 0.020000*np.tanh(( data["loo_ps_ind_05_cat"] +(((( data["ps_ind_15"] * data["loo_ps_ind_18_bin"])* 2.0)* 2.0)* 2.0)))
 v["i71"] = 0.020000*np.tanh(((data["loo_ps_ind_02_cat"] -(data["ps_car_11"] * data["ps_ind_01"])) - data["ps_car_11"]))
v["i72"] = 0.019984*np.tanh(((( -2.0 -(( data["ps_ind_03"] * 2.0)* 2.0)) * data["loo_ps_ind_02_cat"])* 2.0))
v["i73"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] * data["ps_reg_03"])+(data["loo_ps_ind_16_bin"] * data["loo_ps_car_04_cat"])) * 2.0))
v["i74"] = 0.020000*np.tanh(((data["loo_ps_ind_05_cat"] - data["ps_car_13"])*(( data["ps_reg_02"] * 2.0)* 2.0)))
 v["i75"] = 0.019992*np.tanh(((( data["loo_ps_ind_05_cat"] * data["loo_ps_ind_07_bin"])+(data["loo_ps_ind_05_cat"] * data["loo_ps_ind_17_bin"])) - data["loo_ps_car_04_cat"]))
v["i76"] = 0.020000*np.tanh(((( data["loo_ps_car_09_cat"] + data["loo_ps_car_03_cat"])* data["loo_ps_ind_09_bin"])+(data["loo_ps_car_05_cat"] * data["ps_ind_03"])))
 v["i77"] = 0.020000*np.tanh(((data["loo_ps_ind_04_cat"] + data["ps_ind_01"])-(data["ps_ind_01"] * data["ps_ind_15"])))
 v["i78"] = 0.020000*np.tanh(( data["loo_ps_ind_16_bin"] *(( data["ps_ind_01"] - data["ps_reg_01"])+(data["loo_ps_ind_07_bin"] - data["ps_reg_01"]))))
v["i79"] = 0.020000*np.tanh(((( data["loo_ps_ind_07_bin"] * data["ps_ind_15"])+ data["loo_ps_ind_07_bin"])*(data["ps_ind_15"] * 2.0)))
 v["i80"] = 0.019988*np.tanh(((( data["loo_ps_car_11_cat"] + data["ps_car_15"])/2.0)* data["missing"]))
v["i81"] = 0.020000*np.tanh(((( data["ps_ind_03"] *(data["loo_ps_ind_04_cat"] +(data["loo_ps_ind_04_cat"] * 2.0)))+ data["loo_ps_ind_04_cat"])* 2.0))
v["i82"] = 0.020000*np.tanh(( data["ps_car_12"] *(((-(data["ps_car_11"])) * 2.0)* 2.0)))
 v["i83"] = 0.020000*np.tanh(( data["loo_ps_car_01_cat"] *(((data["loo_ps_car_01_cat"] - data["ps_reg_03"])- data["ps_reg_03"])- data["ps_ind_14"])))
 v["i84"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] - data["ps_ind_15"])* data["loo_ps_car_04_cat"])+(data["ps_ind_15"] * data["loo_ps_ind_07_bin"])))
 v["i85"] = 0.020000*np.tanh(( data["loo_ps_car_07_cat"] +(data["loo_ps_ind_02_cat"] +(data["ps_ind_03"] *(data["loo_ps_car_11_cat"] - data["ps_reg_01"])))))
 v["i86"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] + data["loo_ps_ind_05_cat"])/2.0)-(data["ps_reg_01"] *(data["ps_ind_03"] + data["loo_ps_ind_08_bin"]))))
v["i87"] = 0.020000*np.tanh(((((data["loo_ps_car_01_cat"] * data["loo_ps_ind_16_bin"])+ data["loo_ps_ind_16_bin"])/2.0)+(data["loo_ps_car_04_cat"] * data["ps_ind_01"])))
 v["i88"] = 0.020000*np.tanh(((data["loo_ps_ind_02_cat"] + data["ps_ind_03"])*(data["ps_car_12"] -(( data["loo_ps_ind_02_cat"] * 2.0)* 2.0))))
v["i89"] = 0.020000*np.tanh(( data["loo_ps_ind_16_bin"] *(data["ps_reg_02"] +(( data["missing"] + data["loo_ps_car_06_cat"])+ data["loo_ps_car_09_cat"]))))
v["i90"] = 0.020000*np.tanh(( data["loo_ps_ind_06_bin"] *(( data["loo_ps_car_09_cat"] *(( data["loo_ps_ind_06_bin"] + data["loo_ps_car_09_cat"])/2.0)) - data["ps_car_11"])))
 v["i91"] = 0.019984*np.tanh(((( data["loo_ps_car_04_cat"] + data["ps_ind_01"])+ data["loo_ps_car_04_cat"])*(data["loo_ps_ind_05_cat"] - data["ps_reg_03"])))
 v["i92"] = 0.020000*np.tanh(( data["ps_reg_03"] *(data["loo_ps_ind_05_cat"] - data["loo_ps_car_01_cat"])))
 v["i93"] = 0.019988*np.tanh(((((data["ps_car_13"] + data["loo_ps_car_03_cat"])/2.0)+(( data["ps_car_15"] + data["loo_ps_ind_08_bin"])/2.0)) /2.0))
v["i94"] = 0.020000*np.tanh(((( data["loo_ps_ind_05_cat"] * data["ps_reg_03"])- data["ps_reg_01"])*(data["loo_ps_ind_18_bin"] + data["ps_reg_03"])))
 v["i95"] = 0.020000*np.tanh(((data["loo_ps_ind_04_cat"] +(data["ps_ind_15"] *(data["loo_ps_ind_18_bin"] - data["loo_ps_car_06_cat"])))+ data["ps_reg_01"]))
v["i96"] = 0.020000*np.tanh(((data["ps_ind_03"] *(( data["loo_ps_car_05_cat"] +(data["loo_ps_ind_05_cat"] +(data["loo_ps_ind_11_bin"] / 2.0)))/2.0)) / 2.0))
v["i97"] = 0.020000*np.tanh(( data["loo_ps_car_09_cat"] *(data["loo_ps_car_09_cat"] / 2.0)))
 v["i98"] = 0.020000*np.tanh(( data["loo_ps_car_03_cat"] *(( data["loo_ps_car_09_cat"] + data["ps_ind_01"])*(( data["loo_ps_car_04_cat"] + data["ps_ind_01"])/2.0))))
v["i99"] = 0.020000*np.tanh(( data["loo_ps_ind_17_bin"] *(((data["ps_reg_03"] + data["loo_ps_ind_05_cat"])+ data["loo_ps_ind_05_cat"])* 2.0)))
 v["i100"] = 0.019988*np.tanh(((data["ps_car_11"] *(np.tanh(data["loo_ps_ind_04_cat"])* 2.0)) + data["loo_ps_ind_04_cat"]))
v["i101"] = 0.020000*np.tanh(((((data["loo_ps_ind_05_cat"] * data["ps_ind_01"])- data["loo_ps_ind_04_cat"])+ data["loo_ps_ind_18_bin"])* data["ps_ind_01"]))
v["i102"] = 0.020000*np.tanh(((((-1.0 - data["ps_ind_03"])* 2.0)* np.tanh(( data["loo_ps_ind_02_cat"] * 2.0)))* 2.0))
v["i103"] = 0.020000*np.tanh(((((( data["loo_ps_car_09_cat"] + data["loo_ps_ind_07_bin"])/2.0)* data["loo_ps_ind_05_cat"])+(data["loo_ps_ind_17_bin"] * data["loo_ps_car_09_cat"])) /2.0))
v["i104"] = 0.020000*np.tanh(((((-2.0 +(data["ps_ind_03"] * data["ps_ind_03"])) - data["ps_ind_03"])* 2.0)* 2.0))
v["i105"] = 0.020000*np.tanh(((( data["loo_ps_ind_17_bin"] *(data["loo_ps_ind_17_bin"] / 2.0)) * data["loo_ps_car_07_cat"])- data["loo_ps_car_07_cat"]))
v["i106"] = 0.020000*np.tanh(((((data["loo_ps_ind_17_bin"] +(( data["loo_ps_ind_17_bin"] +(data["loo_ps_ind_05_cat"] * data["loo_ps_ind_17_bin"])) /2.0)) /2.0)+ data["loo_ps_ind_12_bin"])/2.0))
v["i107"] = 0.020000*np.tanh(((( data["loo_ps_car_11_cat"] * data["loo_ps_car_03_cat"])- data["loo_ps_car_03_cat"])-(data["ps_car_15"] * data["loo_ps_car_03_cat"])))
 v["i108"] = 0.019996*np.tanh(((( data["loo_ps_car_02_cat"] * data["ps_ind_03"])+(( data["loo_ps_car_02_cat"] +(data["loo_ps_car_11_cat"] / 2.0)) /2.0)) /2.0))
v["i109"] = 0.020000*np.tanh(((( data["ps_reg_03"] + data["loo_ps_car_09_cat"])+ data["ps_ind_01"])*(data["loo_ps_ind_05_cat"] - data["ps_reg_01"])))
 v["i110"] = 0.020000*np.tanh(( np.tanh(((data["loo_ps_car_05_cat"] + data["ps_ind_01"])/2.0)) -(data["ps_car_11"] * data["ps_ind_01"])))
 v["i111"] = 0.020000*np.tanh(( np.tanh(data["loo_ps_ind_04_cat"])*(((-(( data["loo_ps_car_03_cat"] / 2.0)))+ data["loo_ps_ind_04_cat"])/2.0)))
 v["i112"] = 0.020000*np.tanh(((data["loo_ps_car_03_cat"] *(( data["loo_ps_ind_02_cat"] + data["loo_ps_ind_08_bin"])+ data["loo_ps_ind_02_cat"])) * 2.0))
v["i113"] = 0.020000*np.tanh(( data["loo_ps_ind_16_bin"] * data["loo_ps_car_07_cat"]))
v["i114"] = 0.020000*np.tanh(((data["loo_ps_ind_16_bin"] +(((( data["loo_ps_ind_16_bin"] + data["loo_ps_ind_16_bin"])/2.0)+ data["loo_ps_ind_05_cat"])/2.0)) * data["loo_ps_ind_09_bin"]))
v["i115"] = 0.020000*np.tanh(((( data["loo_ps_ind_04_cat"] / 2.0)* data["loo_ps_ind_04_cat"])- data["loo_ps_ind_05_cat"]))
v["i116"] = 0.019871*np.tanh(np.tanh(( data["loo_ps_car_05_cat"] *(data["loo_ps_ind_07_bin"] +(data["loo_ps_ind_02_cat"] * 2.0)))))
 v["i117"] = 0.019996*np.tanh(((( data["loo_ps_car_09_cat"] + data["loo_ps_ind_05_cat"])*(data["loo_ps_ind_07_bin"] + data["loo_ps_car_04_cat"])) - data["loo_ps_ind_17_bin"]))
v["i118"] = 0.019969*np.tanh(((( data["loo_ps_ind_08_bin"] +(( data["ps_car_13"] +(data["loo_ps_ind_12_bin"] / 2.0)) /2.0)) + data["loo_ps_ind_08_bin"])/2.0))
v["i119"] = 0.020000*np.tanh(((data["loo_ps_car_09_cat"] - data["ps_car_15"])*(data["loo_ps_ind_17_bin"] *(data["loo_ps_ind_17_bin"] - data["ps_car_15"]))))
v["i120"] = 0.020000*np.tanh(((( data["loo_ps_ind_02_cat"] *(data["loo_ps_car_09_cat"] + data["ps_car_13"])) * 2.0)* 2.0))
v["i121"] = 0.020000*np.tanh(( data["loo_ps_ind_05_cat"] *(( data["ps_car_12"] + data["ps_car_12"])*(data["loo_ps_car_07_cat"] * 2.0))))
v["i122"] = 0.020000*np.tanh(((( np.tanh(data["loo_ps_ind_07_bin"])/ 2.0)+ np.tanh(( data["loo_ps_ind_07_bin"] / 2.0)))/2.0))
v["i123"] = 0.020000*np.tanh(((data["loo_ps_ind_17_bin"] +(data["loo_ps_ind_07_bin"] *(((data["loo_ps_car_08_cat"] + data["loo_ps_ind_17_bin"])/2.0)+ data["loo_ps_ind_17_bin"])))/2.0))
v["i124"] = 0.019906*np.tanh(((( -(data["loo_ps_car_04_cat"])) * data["ps_car_11"])- data["loo_ps_car_04_cat"]))
v["i125"] = 0.019996*np.tanh(((((((data["loo_ps_car_06_cat"] * data["loo_ps_car_06_cat"])+ data["ps_car_11"])/2.0)* data["loo_ps_car_06_cat"])+ data["loo_ps_car_02_cat"])/2.0))
v["i126"] = 0.020000*np.tanh(( data["ps_reg_03"] *(data["ps_ind_15"] +(( data["loo_ps_ind_05_cat"] + data["ps_ind_15"])+ data["ps_ind_15"]))))
v["i127"] = 0.020000*np.tanh(((((data["loo_ps_car_09_cat"] - data["ps_ind_01"])- data["loo_ps_car_06_cat"])- data["ps_ind_01"])* data["loo_ps_ind_04_cat"]))
v["i128"] = 0.020000*np.tanh(((((data["ps_reg_03"] + data["loo_ps_ind_05_cat"])/2.0)- data["ps_reg_01"])*(data["ps_reg_03"] + data["ps_ind_03"])))
 v["i129"] = 0.020000*np.tanh(((data["loo_ps_car_03_cat"] *(-(data["loo_ps_ind_18_bin"])))+(data["loo_ps_ind_09_bin"] * data["loo_ps_car_09_cat"])))
 v["i130"] = 0.020000*np.tanh(((((data["loo_ps_ind_02_cat"] * data["loo_ps_ind_02_cat"])-(data["loo_ps_ind_04_cat"] * 2.0)) * data["loo_ps_ind_02_cat"])* 2.0))
v["i131"] = 0.019941*np.tanh(((((data["ps_reg_03"] * 2.0)*(data["ps_reg_03"] * data["ps_ind_03"])) * 2.0)- data["ps_ind_03"]))
v["i132"] = 0.020000*np.tanh(((((data["loo_ps_ind_07_bin"] - data["loo_ps_ind_07_bin"])- data["loo_ps_ind_06_bin"])- data["loo_ps_ind_07_bin"])* data["loo_ps_ind_04_cat"]))
v["i133"] = 0.019996*np.tanh(( data["ps_reg_01"] *(((( data["ps_reg_01"] + data["missing"])/2.0)- data["loo_ps_ind_05_cat"])- data["loo_ps_car_09_cat"])))
 v["i134"] = 0.020000*np.tanh(((( data["ps_ind_15"] / 2.0)- data["ps_ind_01"])*(( data["ps_ind_03"] + data["ps_ind_15"])/2.0)))
 v["i135"] = 0.019996*np.tanh(((-(data["ps_car_13"])) -(((( data["ps_car_13"] * data["loo_ps_car_01_cat"])* 2.0)* 2.0)* 2.0)))
 v["i136"] = 0.020000*np.tanh(((( data["ps_reg_03"] +(data["ps_reg_03"] * data["loo_ps_ind_07_bin"])) + data["ps_reg_03"])* data["loo_ps_ind_07_bin"]))
v["i137"] = 0.019988*np.tanh(( data["loo_ps_car_07_cat"] *(data["ps_reg_02"] / 2.0)))
 v["i138"] = 0.020000*np.tanh(( data["loo_ps_ind_02_cat"] *(( data["missing"] +(data["loo_ps_car_09_cat"] * data["loo_ps_ind_02_cat"])) /2.0)))
 v["i139"] = 0.019996*np.tanh(((data["ps_reg_02"] +(data["loo_ps_car_06_cat"] / 2.0)) /2.0))
v["i140"] = 0.020000*np.tanh(((((data["loo_ps_ind_06_bin"] * data["ps_ind_15"])- data["loo_ps_car_06_cat"])* 2.0)* 2.0))
v["i141"] = 0.020000*np.tanh(( data["loo_ps_car_07_cat"] *(( data["loo_ps_ind_05_cat"] * data["ps_car_13"])* data["loo_ps_car_07_cat"])))
 v["i142"] = 0.019980*np.tanh(((data["ps_ind_15"] *(-(data["ps_ind_03"])))- np.tanh(( data["ps_ind_15"] - data["ps_ind_03"]))))
v["i143"] = 0.020000*np.tanh(( data["loo_ps_ind_02_cat"] *(data["ps_car_15"] * data["loo_ps_ind_02_cat"])))
 v["i144"] = 0.019996*np.tanh(((( np.tanh(data["loo_ps_ind_10_bin"])/ 2.0)- data["ps_reg_01"])/ 2.0))
v["i145"] = 0.020000*np.tanh(((( data["missing"] +(data["loo_ps_ind_02_cat"] * data["loo_ps_ind_02_cat"])) + data["loo_ps_car_08_cat"])* data["loo_ps_ind_02_cat"]))
v["i146"] = 0.019984*np.tanh(((( data["ps_reg_02"] - data["loo_ps_car_02_cat"])*(data["ps_reg_01"] * data["loo_ps_car_09_cat"])) / 2.0))
v["i147"] = 0.020000*np.tanh(( data["loo_ps_car_09_cat"] *(( data["loo_ps_car_09_cat"] * 2.0)+(data["loo_ps_car_09_cat"] * 2.0))))
v["i148"] = 0.019977*np.tanh(((data["ps_reg_02"] *(( data["ps_car_13"] - data["ps_reg_03"])* data["loo_ps_ind_06_bin"])) - data["loo_ps_ind_06_bin"]))
v["i149"] = 0.019996*np.tanh(( data["ps_car_11"] *(((data["loo_ps_car_06_cat"] + data["ps_car_11"])/2.0)- data["loo_ps_ind_16_bin"])))
 v["i150"] = 0.020000*np.tanh(((( data["ps_ind_14"] / 2.0)+(data["ps_ind_15"] * data["ps_ind_14"])) /2.0))
return v<feature_engineering>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
traintargets = train.target.values
train = GP1(train)
 train['target'] = traintargets
test = GP1(test )<split>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
xgbtestpreds = np.zeros(test.shape[0])
 scores = 0
myfeatures = list(features)
 kf = StratifiedKFold(n_splits=5,shuffle=True,random_state=2017)
 for i,(train_index, test_index)in enumerate(kf.split(list(train.index),train.target)) :
print('Fold: ',i)
 blindtrain = train.iloc[test_index].copy()
vistrain = train.iloc[train_index].copy()
clf = xgb.XGBClassifier(n_estimators=2000,
learning_rate = 0.04,
max_depth = 5,
min_child_weight = 9,
subsample = 0.8,
colsample_bytree = 0.8,
reg_alpha = 10.4,
reg_lambda = 0.59,
seed = 2017,
nthread = 8,
silent = 1)
 eval_set=[(blindtrain[myfeatures],blindtrain.target)]
model = clf.fit(vistrain[myfeatures], vistrain.target,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=50,
verbose=False)
 print(" Best N trees = ", model.best_ntree_limit)
 print(" Best gini = ", model.best_score)
 trainpreds = model.predict_proba(blindtrain[myfeatures],ntree_limit=model.best_ntree_limit)[:,1]
scores += 0.2*eval_gini(blindtrain.target,trainpreds)
 print(" Best gini = ", eval_gini(blindtrain.target,trainpreds))
xgbtestpreds += model.predict_proba(test[myfeatures],ntree_limit=model.best_ntree_limit)[:,1]
xgbtestpreds /= 5
print(scores )<save_to_csv>
nltk.download('stopwords')
 nltk.download('punkt')
sub = pd.read_csv(strdirectory+'sample_submission.csv')
 sub.target = xgbtestpreds
sub.to_csv('gpxgbhybrid.csv',index=False )<import_modules>
replace_by_space_re = re.compile('[/() {}\[\]\|@,;]')
 bad_symbols_re = re.compile('[^0-9a-z ]')
 links_re = re.compile(r'http\S+')
 Stopwords = set(stopwords.words('english'))
Stopwords.remove('no')
 Stopwords.remove('not')
 lemmatizer = nltk.stem.WordNetLemmatizer()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn import metrics
from sklearn.metrics import roc_auc_score
from sklearn.utils import shuffle
from sklearn.model_selection import StratifiedKFold
import xgboost as xgb<load_from_csv>
def text_prepare(text):

 text = text.lower()
text = re.sub(replace_by_space_re," ",text)
 text = re.sub(bad_symbols_re, "",text)
 text = re.sub(links_re, "",text)
 word_tokens = word_tokenize(text)
 filtered_tokens=[]
for word in word_tokens:
if word not in Stopwords:
filtered_tokens.append(lemmatizer.lemmatize(word))
text = " ".join(word for word in filtered_tokens)
 return text
train = pd.read_csv(".. /input/train.csv", na_values='-1')
 test = pd.read_csv(".. /input/test.csv", na_values='-1')
 print(train.shape)
 print(test.shape )<randomize_order>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
desired_apriori=0.10
idx_0 = train[train.target == 0].index
idx_1 = train[train.target == 1].index
nb_0 = len(train.loc[idx_0])
 nb_1 = len(train.loc[idx_1])
 undersampling_rate =(( 1-desired_apriori)*nb_1)/(nb_0*desired_apriori)
 undersampled_nb_0 = int(undersampling_rate*nb_0)
 print('Rate to undersample records with target=0: {}'.format(undersampling_rate))
print('Number of records with target=0 after undersampling: {}'.format(undersampled_nb_0))
undersampled_idx = shuffle(idx_0, random_state=37, n_samples=undersampled_nb_0)
 idx_list = list(undersampled_idx)+ list(idx_1)
 train = train.loc[idx_list].reset_index(drop=True )<drop_column>
train["text"] = [text_prepare(x)for x in train["text"]]
test["text"] = [text_prepare(x)for x in test["text"]]
vars_to_drop = ['ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_12_bin', 'ps_ind_13_bin', 'ps_reg_03', 'ps_car_03_cat',
'ps_car_05_cat','ps_car_14']
train.drop(vars_to_drop, inplace=True, axis=1)
 test.drop(vars_to_drop, inplace=True, axis=1)
 print(train.shape)
 print(test.shape )<drop_column>
train_input = bert_encode(train["text"], tokenizer, max_len=160)
 test_input = bert_encode(test["text"], tokenizer, max_len=160)
 train_labels = train.target.values
unwanted = train.columns[train.columns.str.startswith('ps_calc_')]
train = train.drop(unwanted, axis=1)
 test = test.drop(unwanted, axis=1 )<create_dataframe>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=5,
callbacks=[checkpoint],
batch_size=16
)
train_data_nato_median = pd.DataFrame()
for column in train.columns:
train_data_nato_median[column] = train[column].fillna(train[column].median())
 train = train_data_nato_median.copy()
test_data_nato_median = pd.DataFrame()
for column in test.columns:
test_data_nato_median[column] = test[column].fillna(test[column].median())
 test = test_data_nato_median.copy()<data_type_conversions>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )
column_names = train.columns
categorical_column = column_names[column_names.str[10] == 'c']
def int_to_categorical(data):

 for column in categorical_column:
data[column] = data[column].astype('category' )<categorify>
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission_BERT.csv', index=False )