Kaggle competition: CSC: HW4 spring19
<SOS> metric: auc Kaggle data source: csc:-hw4-spring19<categorify>
!pip install catboost==0.13.1
tokenizer = Tokenizer(num_words = max_features)
 tokenizer.fit_on_texts(list(X_tr))
X_tr = tokenize_text(X_tr, tokenizer, maxlen)
 X_te = tokenize_text(X_te, tokenizer, maxlen)
 embedding_matrix = create_embedding_matrix(model_embed,
tokenizer,
max_features,
embed_size)
 model = RNN_model(X_tr.shape[1], embed_size, max_features, embedding_matrix)
 model_fited = model.fit(
 X_tr,
Y_train.values,
batch_size = batch_size,
nb_epoch = numb_epoch)
 y_hat_probs = model.predict(X_te)
 y_hat = to_binary(y_hat_probs, 0.38)
 print('The submission file has ' + str(np.sum(y_hat)* 100/len(y_hat)) + ' percent insincere')
 d_test = d_test.reset_index()
d_test['prediction'] = y_hat
to_upload = d_test[['qid', 'prediction']]
to_upload.to_csv('submission.csv', index = False )<import_modules>
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=UserWarning )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
import gc
import numpy as np
import pandas as pd
from time import time
from sklearn.metrics import roc_auc_score, classification_report
from catboost import Pool, CatBoost, CatBoostClassifier, MetricVisualizer
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
basepath = ".. /input/"
df_train = pd.read_csv(f"{basepath}train/train.csv", index_col=0, low_memory=False)
 df_test = pd.read_csv(f"{basepath}test/test.csv", index_col=0, low_memory=False)
 group_col = 'group_id'
target = 'target'
df_train.shape, df_test.shape
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<choose_model_class>
df_train = df_train.drop(df_train.index[[0,1,2,3]])
 df_train = df_train.drop(['feature_13', 'feature_32'], axis=1)
 df_test = df_test.drop(['feature_13', 'feature_32'], axis=1)
 df_train = df_train.rename(columns={'feature_18': 'pos'})
 df_test = df_test.rename(columns={'feature_18': 'pos'} )
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
num_features = list(df_train.select_dtypes(include=[np.float64, np.int64] ).columns)
 num_features.remove(target)
 num_features.remove(group_col)
 mins = df_train[num_features].min()
maxs = df_train[num_features].max()
difs = np.abs(maxs - mins)
 vals = mins - 5 * difs
for col in num_features:
df_train[col] = df_train[col].fillna(vals[col])
 df_test[col] = df_test[col].fillna(vals[col])
 df_train[num_features] = df_train[num_features].astype(np.float32)
 df_test[num_features] = df_test[num_features].astype(np.float32)
 f"{gc.collect() } objects collected"
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def add_aggregates(df, col):
df[f"{col}_min"] = df.groupby(group_col)[col].transform(np.min)
 df[f"{col}_avg"] = df.groupby(group_col)[col].transform(np.mean)
 df[f"{col}_q50"] = df.groupby(group_col)[col].transform(np.median)
 df[f"{col}_max"] = df.groupby(group_col)[col].transform(np.max)
 df[f"{col}_var"] = df.groupby(group_col)[col].transform(np.var)
 df[f"{col}_sum"] = df.groupby(group_col)[col].transform(np.sum)
 df[f"{col}_cmp_min"] = df[col] == df[f"{col}_min"]
df[f"{col}_cmp_max"] = df[col] == df[f"{col}_max"]
df[f"{col}_cmp_avg"] = df[col] <= df[f"{col}_avg"]
df[f"{col}_cmp_q50"] = df[col] <= df[f"{col}_q50"]
df[f"{col}_scaled_min"] = df[col] / df[f"{col}_min"]
df[f"{col}_scaled_avg"] = df[col] / df[f"{col}_avg"]
df[f"{col}_scaled_q50"] = df[col] / df[f"{col}_q50"]
df[f"{col}_scaled_max"] = df[col] / df[f"{col}_max"]
df[f"{col}_scaled_sum"] = df[col] / df[f"{col}_sum"]
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_noemb_val_y>thresh ).astype(int))))<predict_on_test>
cat_features = list(df_train.select_dtypes(include=[np.object] ).columns)
 df_train[cat_features] = df_train[cat_features].fillna('N/A')
 df_test[cat_features] = df_test[cat_features].fillna('N/A')
 all_data = pd.concat(( df_train, df_test))
for col in cat_features:
cats = all_data[col].unique()
df_train[col] = df_train[col].astype('category', categories = cats)
 df_test[col] = df_test[col].astype('category', categories = cats)
 for col in cat_features:
df_train[col] = df_train[col].cat.codes + 1
df_test[col] = df_test[col].cat.codes + 1
del all_data
f"{gc.collect() } objects collected"
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
features = num_features + cat_features
f"Total {len(features)} features: {len(num_features)} num., {len(cat_features)} cat."
del model, inp, x
time.sleep(10 )<statistical_test>
def score_model(model, pool):
ys = pool.get_label()
ys_pred = model.predict(pool)
 results = {'auc': roc_auc_score(ys, ys_pred)}
results.update(classification_report(ys, ys_pred, output_dict=True))
return results
def print_results(r):
if 'time' in r:
print(f"Train time: {r['time'][0]:2d}m {r['time'][1]:2d}s")
 print(f"AUC score: {r['auc']:1.5f}")
 print(f"Class Neg:")
 print(f" precision: {r['0']['precision']:1.5f}")
 print(f" recall: {r['0']['recall']:1.5f}")
 print(f"Class Pos:")
 print(f" precision: {r['1']['precision']:1.5f}")
 print(f" recall: {r['1']['recall']:1.5f}" )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
def make_pool(df, labeled=True):
return Pool(
 data = df[features],
label = df[target] if labeled else None,
cat_features = cat_features,
group_id = df[group_col], 
)
 def space_split(df, val_size=0.1):
df.sort_values([group_col, 'pos'])
 cutoff = df[group_col].iloc[int(( 1 - val_size)* len(df)) ]
pt1, pt2 = df[df[group_col] < cutoff], df[df[group_col] >= cutoff]
return make_pool(pt1), make_pool(pt2 )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def full_train(model, train_data, test_data, name):
train_pool = make_pool(train_data)
 t_start = time()
model.fit(train_pool)
 t_elapsed = int(time() - t_start)
 m, s = t_elapsed // 60, t_elapsed % 60
print(f"Train time: {m}m {s}s")
 if target in test_data.columns:
test_pool = make_pool(test_data)
 return score_model(model, test_pool)
 else:
id_test = test_data.index
xs_test = make_pool(test_data, labeled=False)
 ys_prob = model.predict_proba(xs_test)
 yP_prob = [y2 for y1,y2 in ys_prob]
answer = pd.DataFrame({'Id': id_test, 'target': yP_prob})
 answer.to_csv(f'{name}.csv', sep=',', index=False )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
def validate(model, data, val_size=0.1, rounds=100, fit=True):
results = {}
learn, val = space_split(data, val_size)
 if fit:
t_start = time()
model.fit(
 X=learn,
eval_set=val,
use_best_model=True,
early_stopping_rounds=rounds,
verbose=False,
plot=True 
)
 t_elapsed = int(time() - t_start)
 results['time'] = t_elapsed // 60, t_elapsed % 60
results.update(score_model(model, val))
return results
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
common = {
'loss_function': 'CrossEntropy',
'eval_metric': 'F1',
'od_type': 'IncToDec',
'od_pval': 1e-5,
'boosting_type': 'Ordered',
'bootstrap_type': 'Bernoulli',
'one_hot_max_size': 3,
'random_seed': 51,
'task_type': 'GPU',
}
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
params = {
**common, 'train_dir': 'p08-2naf',
'iterations': 1500,
'learning_rate': 0.05,
'depth': 8,
'l2_leaf_reg': 12.0,
'random_strength': 3.0,
'subsample': 0.5,
}
<train_model><EOS> 
model = CatBoostClassifier(**params)
 full_train(model, df_train, df_test, 'm13-kernel-catboost' )