Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<init_hyperparams>
import numpy as np
import pandas as pd
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 50000
n_splits = 10<drop_column>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv" )
X_train = X_train.drop('img_Unnamed: 0',axis=1)
 X_test = X_test.drop('img_Unnamed: 0',axis=1 )<prepare_x_and_y>
data = pd.concat([train, test], sort=False )
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train_lgb = np.zeros(( X_train.shape[0]))
oof_test_lgb = np.zeros(( X_test.shape[0], n_splits))
qwk_scores = []
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(val_pred, y_val)
 coefficients = optR.coefficients()
pred_val_y_k = optR.predict(val_pred, coefficients)
 print("Valid Counts = ", Counter(y_val))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(y_val, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train_lgb[valid_index] = val_pred
oof_test_lgb[:, i] = test_pred
i += 1
print('{} cv mean QWK score : {}'.format('LGBM', np.mean(qwk_scores)) )<features_selection>
data.isnull().sum()
importance_type= "split"
idx_sort = np.argsort(model.feature_importance(importance_type=importance_type)) [::-1]
names_sorted = np.array(model.feature_name())[idx_sort]
imports_sorted = model.feature_importance(importance_type=importance_type)[idx_sort]
for n, im in zip(names_sorted, imports_sorted):
print(n, im )<compute_train_metric>
data.isnull().sum()
optR = OptimizedRounder()
optR.fit(oof_train_lgb, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train_lgb, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk )<predict_on_test>
data['Sex'].replace(['male','female'], [0, 1], inplace=True )
coefficients_ = coefficients.copy()
coefficients_[0] = 1.79
coefficients_[1] = 2.39
coefficients_[3] = 2.99
train_predictions_lgb = optR.predict(oof_train_lgb, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions_lgb)))
 test_predictions_lgb = optR.predict(oof_test_lgb.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions_lgb)) )<count_values>
data['Embarked'].fillna(( 'S'), inplace=True)
 data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
print("True Distribution:")
 print(pd.value_counts(X_train['AdoptionSpeed'], normalize=True ).sort_index())
 print("
Train Predicted Distribution:")
 print(pd.value_counts(train_predictions_lgb, normalize=True ).sort_index())
 print("
Test Predicted Distribution:")
 print(pd.value_counts(test_predictions_lgb, normalize=True ).sort_index() )<concatenate>
data['Fare'].fillna(np.mean(data['Fare']), inplace=True)
 data['fare_value']=data['Fare']/50
df = pd.concat([X_train, X_test], axis=0)
 df.head(2 )<concatenate>
age_avg = data['Age'].mean()
age_std = data['Age'].std()
data['Age'].fillna(np.random.randint(age_avg - age_std, age_avg + age_std), inplace=True)
 data['age_value']=data['Age']/50
df_ = pd.concat([train,test],axis=0 )<categorify>
data['family'] =(data['SibSp'] + data['Parch'])/5
word_vec_size = 300
max_words = 100
max_word_features = 25000
def transform_text(text, tokenizer):
tokenizer.fit_on_texts(text)
 text_emb = tokenizer.texts_to_sequences(text)
 text_emb = sequence.pad_sequences(text_emb, maxlen=max_words)
 return text_emb
desc_tokenizer = text.Tokenizer(num_words=max_word_features)
 desc_embs = transform_text(df_["Description"].astype(str), desc_tokenizer )<define_variables>
data['isAlone'] = 0
data.loc[data['family'] > 0, 'isAlone'] = 1
text_mode = "fasttext"
if text_mode == "fasttext":
embedding_file = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(embedding_file))
word_index = desc_tokenizer.word_index
print('Word index len:', len(word_index))
vocabulary_size = min(max_word_features, len(word_index)) + 1
text_embs = np.zeros(( vocabulary_size, word_vec_size))
for word, i in word_index.items() :
if i >= max_word_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: text_embs[i] = embedding_vector
del(embeddings_index)
 elif text_mode == "word2vec":
embedding_file = ".. /input/word2vec-google/GoogleNews-vectors-negative300.bin"
print("Loading word vectors...")
 word_vectors = KeyedVectors.load_word2vec_format(embedding_file, binary=True)
 print("Matching word vectors...")
 EMBEDDING_DIM=300
word_index = desc_tokenizer.word_index
vocabulary_size=min(len(word_index)+1,max_word_features)
 text_embs = np.zeros(( vocabulary_size, EMBEDDING_DIM))
for word, i in word_index.items() :
if i>=max_word_features:
continue
try:
embedding_vector = word_vectors[word]
text_embs[i] = embedding_vector
except KeyError:
text_embs[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)
 del(word_vectors )<count_missing_values>
delete_columns = ['Name','PassengerId','SibSp','Parch','Ticket','Cabin','Age','Fare']
data.drop(delete_columns, axis=1, inplace=True )
np.sum(pd.isnull(df))<define_variables>
train = data[:len(train)]
test = data[len(train):]
cat_vars = ["Type", "Breed1", "Breed2", "Color1", "Color2", "Color3", "Gender", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "Health", "State"]
cont_vars = ["Fee", "PhotoAmt", "VideoAmt", "Age", "Quantity",'RescuerID_CNT']<categorify>
y_train0 = train['Survived']
X_train0 = train.drop('Survived', axis = 1)
 X_test0 = test.drop('Survived', axis = 1 )
def preproc(df):
global cont_vars
for var in cat_vars:
df[var] = LabelEncoder().fit_transform(df[var])
 for var in cont_vars:
df[var] = MinMaxScaler().fit_transform(df[var].values.reshape(-1,1))
return df<normalization>
X = np.array(X_train0)
 y = np.array(y_train0 )
df_scaled = preproc(df)
 train_df = df_scaled[:len(train)]
test_df = df_scaled[len(train):]
len(train_df), len(test_df )<feature_engineering>
clf = XGBClassifier(max_depth=3, n_estimators=1000, learning_rate=0.01 )
def get_keras_data(df, description_embeds):
X = {var: df[var].values for var in cont_vars+cat_vars}
X["description"] = description_embeds
for i in range(256): X[f"img_{i}"] = df[f"img_{i}"]
return X<import_modules>
ss = ShuffleSplit(n_splits=5,
train_size=0.8,
test_size =0.2,
random_state=0)
 for train_index, test_index in ss.split(X):
X_train, X_test = X[train_index], X[test_index]
Y_train, Y_test = y[train_index], y[test_index]
clf.fit(X_train, Y_train)
 print(clf.score(X_test, Y_test))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class QWKEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.history = []
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, batch_size=1000, verbose=0)
 y_pred = eval_predict(self.y_val, y_pred)
 score = quadratic_weighted_kappa(self.y_val, y_pred)
 print("QWK - epoch: %d - score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history): self.model.save('checkpoint.h5')
 def eval_predict(y=[], y_pred=[], coeffs=None, ret_coeffs=False):
optR = OptimizedRounder()
if not coeffs:
optR.fit(y_pred.reshape(-1,), y)
 coeffs = optR.coefficients()
if ret_coeffs: return optR.coefficients()
return optR.predict(y_pred, coeffs ).reshape(-1, )<compute_test_metric>
y_pred = clf.predict(np.array(X_test0))
def rmse(y, y_pred):
return K.sqrt(K.mean(K.square(y-y_pred), axis=-1))
def get_model(emb_n=10, dout=.25, batch_size=1000):
inps = []
embs = []
nums = []
for var in cat_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 embs.append(( Embedding(df[var].max() +1, emb_n )(inp)))
 for var in cont_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 nums.append(( inp))
desc_inp = Input(shape=(max_words,), name="description")
 inps.append(desc_inp)
 emb_desc = Embedding(vocabulary_size, word_vec_size, weights=[text_embs] )(desc_inp)
 emb_desc = SpatialDropout1D (.4 )(emb_desc)
 emb_desc = Reshape(( max_words, word_vec_size, 1))(emb_desc)
 filter_sizes=[1,3,1]
convs = []
for filter_size in filter_sizes:
conv = Conv2D(32, kernel_size=(filter_size, word_vec_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool2D(pool_size=(max_words-filter_size+1, 1))(conv))
img_fts = []
for i in range(256):
inp = Input(shape=[1], name=f"img_{i}")
 inps.append(inp)
 img_fts.append(( inp))
img_fts = concatenate(img_fts)
 img_fts = Dropout (.5 )(img_fts)
 img_fts = BatchNormalization()(img_fts)
 img_fts = Dense(64, activation="relu" )(img_fts)
 convs = concatenate(convs)
 convs = Flatten()(convs)
 convs = BatchNormalization()(convs)
 embs = Flatten()(concatenate(embs))
embs = Dropout(dout )(Dense(64, activation="relu", kernel_initializer="he_normal" )(embs))
nums = concatenate(nums)
 nums = Dense(32, activation="relu" )(nums)
 nums = Dropout(0.5 )(nums)
 x = concatenate([embs, nums, convs, img_fts])
 x = BatchNormalization()(x)
 dense_n = [256, 64]
for n in dense_n:
x = BatchNormalization()(x)
 x = Dense(n, activation="relu", kernel_initializer="he_normal" )(x)
 x = BatchNormalization()(x)
 x = Dropout (.2 )(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inps, outputs=out)
 opt = Adam()
model.compile(optimizer=opt, loss='mse',)
 return model 
<prepare_x_and_y>
sub = gender_submission
sub['Survived'] = list(map(int, y_pred))
sub.to_csv("submission.csv", index=False )