Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.python.framework import ops
import matplotlib.pyplot as plt
if TRAIN_MODE:
y_true2 = y_true.numpy()
y_pred2 = y_pred.numpy()
labels = df_submission.columns[1:].tolist()
label_size = len(labels)
 P = [None] * label_size
R = [None] * label_size
AP = np.zeros(label_size)
 for i in range(label_size):
P[i], R[i], _ = precision_recall_curve(y_true2[:,i], y_pred2[:,i])
 AP[i] = average_precision_score(y_true2[:,i], y_pred2[:,i])
 df_ap = pd.DataFrame(data=AP, index=labels, columns=['AP'])
 df_ap.to_csv('labels_ap.csv')
 P_micro, R_micro, _ = precision_recall_curve(y_true2.ravel() , y_pred2.ravel())
 AP_micro = average_precision_score(y_true2, y_pred2, average='micro')
 fig = plt.figure(figsize=(8,11))
ax = fig.add_subplot(1,1,1)
 df_ap.sort_values('AP' ).plot.barh(
 title='Average Precision per Labels(μAP=%.5f)' % AP_micro,
grid=True, legend=False, xlim=(0.,1.) , ax=ax, 
)
 plt.tight_layout()
plt.show()<load_pretrained>
def read_data(file_name):

 data = pd.read_csv('.. /input/'+file_name+'.csv')
 return data
MPLoader.reset()
MPLoader.full_load(df_test, use_preprocess=False )<load_pretrained>
def prepare_age(data):

 age = data['Age']
mean_age = age.mean()
var_age = age.var()
age[age.isnull() ] = mean_age
age = age - mean_age
age = age / var_age
return age.as_matrix()
USE_MASK_FREQ = USE_MASK_TIME = False
test = ImageList.from_df(df_test, WORK, folder='')
 learn = load_learner(WORK, test=test)if TRAIN_MODE else load_learner('.', DEPLOYED_MODEL, test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test, num_pred=50 )<save_to_csv>
def prepare_fare(data):

 fare = data['Fare']
mean_fare = fare.mean()
var_fare = fare.var()
fare = fare - mean_fare
fare = fare / var_fare
return fare.as_matrix()
df_submission[learn.data.classes] = preds
df_submission.to_csv('submission.csv', index=False)
 df_submission.head()<set_options>
def prepare_sex(data):

 sex = data['Sex']
sex = np.where(sex=='male',0,1)
 return sex
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )<categorify>
def prepare_embarquation(data):

 embarked = data['Embarked']
embarked[embarked.isnull() ] = 3
embarked = np.where(embarked=='C', 0, embarked)
 embarked = np.where(embarked=='Q', 1, embarked)
 embarked = np.where(embarked=='S', 2, embarked)
 mean_embarked = embarked.mean()
var_embarked = embarked.var()
embarked = embarked - mean_embarked
embarked = embarked / var_embarked
return embarked
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
def prepare_sibligs(data):

 sib = data['SibSp']
mean_sib = sib.mean()
var_sib = sib.var()
sib = sib - mean_sib
sib = sib / var_sib
return sib
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED = Path('.. /input/fat2019_prep_mels1')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_TRN_NOISY_BEST50S = PREPROCESSED/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
MELS_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'
MELS_TRN_NOISY = PREPROCESSED/'mels_train_noisy.pkl'
MELS_TRN_NOISY_BEST50S = PREPROCESSED/'mels_trn_noisy_best50s.pkl'
MELS_TEST = PREPROCESSED/'mels_test.pkl'
trn_curated_df = pd.read_csv(CSV_TRN_CURATED)
 trn_noisy_df = pd.read_csv(CSV_TRN_NOISY)
 trn_noisy50s_df = pd.read_csv(CSV_TRN_NOISY_BEST50S)
 test_df = pd.read_csv(CSV_SUBMISSION)
 df = pd.concat([trn_curated_df, trn_noisy50s_df], ignore_index=True, sort=True)
 test_df = pd.read_csv(CSV_SUBMISSION)
 X_train = pickle.load(open(MELS_TRN_CURATED, 'rb')) + pickle.load(open(MELS_TRN_NOISY_BEST50S, 'rb'))<define_variables>
def prepare_parch(data):

 parch = data['Parch']
mean_parch = parch.mean()
var_parch = parch.var()
parch = parch - mean_parch
parch = parch / var_parch
return parch
data.show_batch(3 )<compute_test_metric>
def prepare_family_size(data):

 parch = data['Parch']
sib = data['SibSp']
family_size = parch + sib
mean_family_size = family_size.mean()
var_family_size = family_size.var()
family_size = family_size - mean_family_size
family_size = family_size / var_family_size
return family_size
def lwlrap(y_pred,y_true):
score, weight = calculate_per_class_lwlrap(y_true.cpu().numpy() , y_pred.cpu().numpy())
 lwlrap =(score * weight ).sum()
return torch.from_numpy(np.array(lwlrap))<train_model>
def normalize_features(data):

 age = prepare_age(data)
 sex = prepare_sex(data)
 embark = prepare_embarquation(data)
 fare = prepare_fare(data)
 sib = prepare_sibligs(data)
 parch = prepare_parch(data)
 family_size = prepare_family_size(data)
 X_train = np.column_stack(( sex, age, family_size, embark))
return X_train.T
class MixUpCallback(LearnerCallback):
"Callback that creates the mixed-up input and target."
def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):
super().__init__(learn)
 self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y
def on_train_begin(self, **kwargs):
if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)
 def on_batch_begin(self, last_input, last_target, train, **kwargs):
"Applies mixup to `last_input` and `last_target` if `train`."
if not train: return
lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))
lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1 ).max(1)
 lambd = last_input.new(lambd)
 shuffle = torch.randperm(last_target.size(0)).to(last_input.device)
 x1, y1 = last_input[shuffle], last_target[shuffle]
if self.stack_x:
new_input = [last_input, last_input[shuffle], lambd]
else:
new_input =(last_input * lambd.view(lambd.size(0),1,1,1)+ x1 *(1-lambd ).view(lambd.size(0),1,1,1))
if self.stack_y:
new_target = torch.cat([last_target[:,None].float() , y1[:,None].float() , lambd[:,None].float() ], 1)
 else:
if len(last_target.shape)== 2:
lambd = lambd.unsqueeze(1 ).float()
new_target = last_target.float() * lambd + y1.float() *(1-lambd)
 return {'last_input': new_input, 'last_target': new_target}
def on_train_end(self, **kwargs):
if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()
class MixUpLoss(nn.Module):
"Adapt the loss function `crit` to go with mixup."
def __init__(self, crit, reduction='mean'):
super().__init__()
if hasattr(crit, 'reduction'):
self.crit = crit
self.old_red = crit.reduction
setattr(self.crit, 'reduction', 'none')
 else:
self.crit = partial(crit, reduction='none')
 self.old_crit = crit
self.reduction = reduction
def forward(self, output, target):
if len(target.size())== 2:
loss1, loss2 = self.crit(output,target[:,0].long()), self.crit(output,target[:,1].long())
 d =(loss1 * target[:,2] + loss2 *(1-target[:,2])).mean()
else: d = self.crit(output, target)
 if self.reduction == 'mean': return d.mean()
elif self.reduction == 'sum': return d.sum()
return d
def get_old(self):
if hasattr(self, 'old_crit'): return self.old_crit
elif hasattr(self, 'old_red'):
setattr(self.crit, 'reduction', self.old_red)
 return self.crit
def mixup(learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True)-> Learner:
"Add mixup https://arxiv.org/abs/1710.09412 to `learn`."
learn.callback_fns.append(partial(MixUpCallback, alpha=alpha, stack_x=stack_x, stack_y=stack_y))
return learn
Learner.mixup = mixup 
<define_search_model>
def prepare_training_data() :

 pd.set_option('mode.chained_assignment', None)
 data = read_data('train')
 X_train = normalize_features(data)
 Y_train = np.reshape(data['Survived'].as_matrix() ,(X_train.shape[1],1)).T
return X_train, Y_train
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
class Classifier(nn.Module):
def __init__(self, num_classes=1000):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = self.avgpool(x)
 x = self.fc(x)
 return x<choose_model_class>
def prepare_test_data() :

 pd.set_option('mode.chained_assignment', None)
 data = read_data('test')
 X_test = normalize_features(data)
 return X_test
def borrowed_model(pretrained=False, **kwargs):
return Classifier(**kwargs)
 f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(data, borrowed_model, pretrained=False, metrics=[lwlrap] ).mixup(stack_y=False)
 learn.unfreeze() 
<train_model>
def initialize_Parameters(nb_features):

 W1 = tf.get_variable("W1", [5, nb_features], initializer = tf.contrib.layers.xavier_initializer())
 b1 = tf.get_variable("b1", [5,1], initializer = tf.zeros_initializer())
 W2 = tf.get_variable("W2", [8,5], initializer = tf.contrib.layers.xavier_initializer())
 b2 = tf.get_variable("b2", [8,1], initializer = tf.zeros_initializer())
 W3 = tf.get_variable("W3", [6,8], initializer = tf.contrib.layers.xavier_initializer())
 b3 = tf.get_variable("b3", [6,1], initializer = tf.zeros_initializer())
 W4 = tf.get_variable("W4", [4,6], initializer = tf.contrib.layers.xavier_initializer())
 b4 = tf.get_variable("b4", [4,1], initializer = tf.zeros_initializer())
 W5 = tf.get_variable("W5", [1,4], initializer = tf.contrib.layers.xavier_initializer())
 b5 = tf.get_variable("b5", [1,1], initializer = tf.zeros_initializer())
 parameters = {"W1": W1, "b1": b1,
"W2": W2, "b2": b2,
"W3": W3, "b3": b3,
"W4": W4, "b4": b4,
"W5": W5, "b5": b5
}
return parameters
learn.fit_one_cycle(255, 1e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )<train_model>
def forward_propagation(X, parameters):

 W1 = parameters['W1']
b1 = parameters['b1']
W2 = parameters['W2']
b2 = parameters['b2']
W3 = parameters['W3']
b3 = parameters['b3']
W4 = parameters['W4']
b4 = parameters['b4']
W5 = parameters['W5']
b5 = parameters['b5']
Z1 = tf.add(tf.matmul(W1, X), b1)
 A1 = tf.nn.relu(Z1)
 Z2 = tf.add(tf.matmul(W2, A1), b2)
 A2 = tf.nn.relu(Z2)
 Z3 = tf.add(tf.matmul(W3, A2), b3)
 A3 = tf.nn.relu(Z3)
 Z4 = tf.add(tf.matmul(W4, A3), b4)
 A4 = tf.nn.relu(Z4)
 Z5 = tf.add(tf.matmul(W5, A4), b5)
 return Z5
learn.lr_find()
learn.fit_one_cycle(50, 1e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )<load_from_csv>
def create_placeholders(n_x, n_y):

 X = tf.placeholder(dtype=tf.float32, shape=([n_x, None]), name="X")
 Y = tf.placeholder(dtype=tf.float32, shape=([n_y, None]), name="Y")
 return X, Y
del X_train
X_test = pickle.load(open(MELS_TEST, 'rb'))
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test )<save_to_csv>
def compute_cost(Z, Y):

 logits = tf.transpose(Z)
 labels = tf.transpose(Y)
 cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=labels))
return cost
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
def train_predict_model(learning_rate, epoch, X_train, Y_train, X_test):

 X, Y = create_placeholders(X_train.shape[0], Y_train.shape[0])
 parameters = initialize_Parameters(X_train.shape[0])
 Z4 = forward_propagation(X, parameters)
 cost = compute_cost(Z4, Y)
 optimizer = tf.train.AdamOptimizer(learning_rate ).minimize(cost)
 costs = []
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 for i in range(1,epoch):
_ , calculated_cost = sess.run([optimizer, cost], feed_dict={X: X_train, Y: Y_train})
 if i % 50 == 0:
costs.append(calculated_cost)
 plot_cost(costs, learning_rate)
 parameters = sess.run(parameters)
 y_pred_proba = tf.cast(tf.nn.sigmoid(Z4),dtype = tf.float32);
y_pred_class = tf.cast(tf.greater(y_pred_proba, 0.5),'float')
 accuracy = tf.reduce_mean(tf.cast(tf.equal(y_pred_class, Y), 'float'))
prediction=tf.argmax(tf.nn.sigmoid(Z4))
sess.run([prediction], feed_dict={X: X_train, Y: Y_train})
 print("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
prediction_test = sess.run(y_pred_class, feed_dict={X:X_test})
 return prediction_test
<set_options><EOS> 
ops.reset_default_graph()
X_train, Y_train = prepare_training_data()
X_test = prepare_test_data()
prediction_test = train_predict_model(learning_rate=0.0001, epoch=40000, X_train= X_train
, Y_train = Y_train, X_test = X_test)
 prediction_test = np.where(prediction_test < 1, 0, 1)
 data = read_data('test')
 submission = pd.DataFrame({
"PassengerId": data["PassengerId"],
"Survived": prediction_test.reshape(X_test.shape[1])
 })
 submission.to_csv('submission.csv', index=False)
 print(check_output(["ls", "."] ).decode("utf8"))