Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<save_to_csv>
%matplotlib inline
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<set_options>
df_train=pd.read_csv('.. /input/train.csv')
 df_test=pd.read_csv('.. /input/test.csv' )
tf.reset_default_graph()<load_from_csv>
def check_missing_data(df):
total=df.isnull().sum().sort_values(ascending=False)
 percent=(( df.isnull().sum() /df.isnull().count())*100 ).sort_values(ascending=False)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'] )
df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 df = df[:1000]<define_variables>
df_tmp=pd.DataFrame(df_train.nunique().sort_values() ,columns=['num_unique_values'] ).reset_index().rename(columns={'index':'Column_name'})
 df_tmp.head()
class FLAGS(object):
bert_config_file = '.. /input/bert_files/bert_config.json'
vocab_file = '.. /input/bert_files/vocab.txt'
model_dir = ".. /input/toxic1"
do_lower_case = True,
max_seq_length = 128
class InputFeatures(object):
def __init__(self, input_ids, input_mask, segment_ids, label_id, is_real_example=True):
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.label_id = label_id<string_transform>
def col_name_with_n_unique_value(df,n):
df1=pd.DataFrame(df.nunique().sort_values() ,columns=['num_unique_values'] ).reset_index()
col_name=list(df1[df1.num_unique_values==1]['index'])
 print('number of columns with only',n,'unique values are: ',len(col_name))
return col_name
def convert_single_example(ex_index, text, label_list, max_seq_length, tokenizer):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)> max_seq_length - 2:
offset = int(max_seq_length / 2)- 1
tokens_a = tokens_a[:offset] + tokens_a[-offset:]
tokens = ["[CLS]"] + tokens_a + ['[SEP]']
input_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_mask = [1] * len(input_ids)
 while len(input_ids)< max_seq_length:
input_ids.append(0)
 input_mask.append(0)
 segment_ids = [0] * len(input_ids)
 if ex_index < 3:
tf.logging.info("*** Examples ***")
 tf.logging.info("tokens: %s" % " ".join([tokenization.printable_text(x)for x in tokens]))
tf.logging.info("input_ids: %s" % " ".join([str(x)for x in input_ids]))
tf.logging.info("input_mask: %s" % " ".join([str(x)for x in input_mask]))
tf.logging.info("segment_ids: %s" % " ".join([str(x)for x in segment_ids]))
feature = InputFeatures(
 input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=0 
)
 return feature<categorify>
col_to_drop=col_name_with_n_unique_value(df_train,1 )
def file_based_convert_examples_to_features(df, label_list, max_seq_length, tokenizer, output_file):
writer = tf.python_io.TFRecordWriter(output_file)
 print(len(df))
for ex_index in range(len(df)) :
if ex_index % 5000 == 0:
tf.logging.info("Writing example %d of %d" %(ex_index, len(df)))
 row = df.iloc[ex_index]
feature = convert_single_example(ex_index, row['comment_text'], label_list,
max_seq_length, tokenizer)
 def create_int_feature(values):
f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))
 return f
features = collections.OrderedDict()
features["input_ids"] = create_int_feature(feature.input_ids)
 features["input_mask"] = create_int_feature(feature.input_mask)
 features["segment_ids"] = create_int_feature(feature.segment_ids)
 features["label_ids"] = create_int_feature([feature.label_id])
 tf_example = tf.train.Example(features=tf.train.Features(feature=features))
writer.write(tf_example.SerializeToString())
 writer.close()<prepare_x_and_y>
del df_train,df_test,df_tmp
gc.collect()
def file_based_input_fn_builder(input_file, seq_length):
name_to_features = {
"input_ids": tf.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.FixedLenFeature([seq_length], tf.int64),
"label_ids": tf.FixedLenFeature([], tf.int64),
}
def input_fn(params):
d = tf.data.TFRecordDataset(input_file)
 d = d.apply(
 tf.contrib.data.map_and_batch(
 lambda record: tf.parse_single_example(record, name_to_features),
batch_size=8,
drop_remainder=False))
return d
return input_fn 
<choose_model_class>
sc = MinMaxScaler(feature_range=(0,1))
train_sc = sc.fit_transform(train)
 test_sc = sc.transform(test )
def create_infer_model(input_ids, input_mask, segment_ids, labels, num_labels):
model = modeling.BertModel(
 config=modeling.BertConfig.from_json_file(FLAGS.bert_config_file),
is_training=False,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=segment_ids,
use_one_hot_embeddings=False)
 output_layer = model.get_pooled_output()
hidden_size = output_layer.shape[-1].value
output_weights = tf.get_variable(
 "output_weights", [num_labels, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable(
 "output_bias", [num_labels], initializer=tf.zeros_initializer())
 logits = tf.matmul(output_layer, output_weights, transpose_b=True)
 logits = tf.nn.bias_add(logits, output_bias)
 probabilities = tf.nn.sigmoid(logits)
 return probabilities<find_best_params>
X_train, X_val, y_train, y_val = train_test_split(train_sc, t, test_size=0.2, random_state=0)
 print(X_train.shape, y_train.shape)
 print(X_val.shape, y_val.shape)
 print(test_sc.shape, test_sc.shape )
def infer_fn_builder(num_labels):
def model_fn(features):
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
is_real_example = None
probabilities = create_infer_model(
 input_ids, input_mask, segment_ids, label_ids, num_labels)
 output_spec = tf.estimator.EstimatorSpec(
 predictions={"probabilities": probabilities},
mode='infer' 
)
 return output_spec
return model_fn<choose_model_class>
del train_sc,train,test
gc.collect()
tic = time.time()
tf.logging.set_verbosity(tf.logging.INFO)
 label_list = ["0", "1"]
tokenizer = tokenization.FullTokenizer(
 vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
 model_fn = infer_fn_builder(num_labels=len(label_list))
estimator = tf.estimator.Estimator(
 model_fn=model_fn,
config=tf.estimator.RunConfig(model_dir=FLAGS.model_dir)
)
 if not os.path.isfile("predict.tf_record"):
file_based_convert_examples_to_features(df, label_list, FLAGS.max_seq_length,
tokenizer, "predict.tf_record")
 predict_input_fn = file_based_input_fn_builder("predict.tf_record", FLAGS.max_seq_length)
 result = estimator.predict(input_fn=predict_input_fn)
 predictions = []
for pred in result:
predictions.append(pred['probabilities'])
 print(f'{len(predictions)} records done in {time.time() - tic}s')
 print(predictions[:10] )<save_to_csv>
Model_Summary=pd.DataFrame()
out = pd.DataFrame(predictions)
 out.columns = ['civil','toxic']
df['prediction'] = out['toxic']
submission = df[['id', 'prediction']]
submission.to_csv('submission.csv', index=False)
 submission.head(10 )<import_modules>
train_data=lightgbm.Dataset(X_train,y_train)
 valid_data=lightgbm.Dataset(X_val,y_val )
np.random.seed(2020)
<load_from_csv>
params={'learning_rate':0.01,
'boosting_type':'gbdt',
'objective':'regression',
'metric':'rmse',
'sub_feature':0.5,
'num_leaves':180,
'feature_fraction': 0.5,
'bagging_fraction': 0.5,
'min_data':50,
'max_depth':-1,
'reg_alpha': 0.3,
'reg_lambda': 0.1,
'min_child_weight': 10,
'verbose': 1,
'nthread':5,
'max_bin':512,
'subsample_for_bin':200,
'min_split_gain':0.0001,
'min_child_samples':5
}
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df.head()<count_values>
lgbm = lightgbm.train(params,
train_data,
25000,
valid_sets=valid_data,
early_stopping_rounds= 80,
verbose_eval= 10
)
df['target'].value_counts()<data_type_conversions>
pred_lgbm=np.expm1(lgbm.predict(test_sc))
pred_lgbm
df[['keyword', 'location']] = df[['keyword', 'location']].astype('str')
 df.info()<load_pretrained>
from catboost import CatBoostRegressor
nlp = spacy.load('en_core_web_lg' )<define_variables>
cb_model = CatBoostRegressor(iterations=500,
learning_rate=0.1,
depth=5,
l2_leaf_reg=20,
bootstrap_type='Bernoulli',
subsample=0.6,
eval_metric='RMSE',
random_seed = 42,
od_type='Iter',
metric_period = 50,
od_wait=45 )
with nlp.disable_pipes() :
text_vectors = np.array([nlp(text ).vector for text in df['text']] )<define_variables>
cb_model.fit(X_train, y_train,
eval_set=(X_val, y_val),
use_best_model=True,
verbose=True )
labels = df['target']<split>
pred_cb=np.expm1(cb_model.predict(test_sc))
pred_cb
X_train, X_valid, y_train, y_valid = train_test_split(text_vectors, labels, test_size=0.2, random_state=3 )<choose_model_class>
from sklearn.ensemble import RandomForestRegressor
mlp_model = Sequential([
Dense(64, input_dim=input_shape, activation='relu'),
Dropout(0.2),
Dense(32, activation='relu'),
Dense(1, activation='sigmoid'),
])
 mlp_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
from sklearn.ensemble import RandomForestRegressor
mlp_model.fit(X_train, y_train,
batch_size=32,
epochs=30,
verbose=2,
callbacks=callbacks,
validation_data=(X_valid, y_valid))<compute_test_metric>
rf_model = RandomForestRegressor(n_estimators=100 )
test_score = mlp_model.evaluate(X_valid, y_valid)
 print(test_score )<load_from_csv>
rf_model.fit(X_train, y_train )
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 with nlp.disable_pipes() :
test_text_vectors = np.array([nlp(text ).vector for text in test['text']] )<save_to_csv>
pred_rf=np.expm1(rf_model.predict(test_sc))
pred_rf
test_labels = mlp_model.predict_classes(test_text_vectors)
 submit_data = pd.DataFrame(test['id'])
 submit_data['target'] = test_labels
submit_data.to_csv('nlp_submit.csv', index=False )<import_modules>
from xgboost import XGBRegressor
%matplotlib inline
cf.go_offline()
cf.set_config_file(offline=False, world_readable=True)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
xgb_model=XGBRegressor(max_depth=9 )
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 train_df.head()<feature_engineering>
xgb_model.fit(X_train, y_train )
"{:.2f}% of the entries which represent actual disasters have a location".format(( len(train_df[(train_df['haslocation'] == 1)&(train_df['target'] == 1)])/ len(train_df[(train_df['target'] == 1)])* 100))<feature_engineering>
pred_xgb=np.expm1(xgb_model.predict(test_sc))
pred_xgb
"{:.2f}% of the entries which do not represent disasters have a location".format(( len(train_df[(train_df['haslocation'] == 1)&(train_df['target'] == 0)])/ len(train_df[(train_df['target'] == 0)])* 100))<drop_column>
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub.target=(pred_lgbm+pred_cb+pred_rf+pred_xgb)/4.0
sub.head()
<feature_engineering><EOS> 
sub.to_csv('sub1.csv',index=False )