Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
!pip install pyspark
matrixes = [embedding_matrix,embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix,embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()<import_modules>
spark = SparkSession.builder.appName('classification' ).getOrCreate()
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
batch_size = int(x_train.shape[0]*K.eval(adam.lr))
print('Batch size = ',batch_size)
 inp1 = Input(shape=(max_len,))
x = Embedding(num_words, dim, weights=[matrix],trainable = False, )(inp1)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(50, return_sequences=True))(x)
 atten_1 = Attention(max_len )(x)
 atten_2 = Attention(max_len )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 x = concatenate([atten_1, atten_2, avg_pool, max_pool])
 x = Dense(100, activation="relu",kernel_regularizer=regularizers.l2(0.0001))(x)
 x = Dropout(0.2 )(x)
 x = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy',f1])
 print(model.summary())
 patience = 10
history = model.fit(x_train,y_train,
batch_size=batch_size,
validation_split=0.33,
epochs=100,
callbacks=[EarlyStopping(patience=patience,monitor='f1'),])
 print('training finished...' )<predict_on_test>
from itertools import chain
from pyspark.sql.functions import count, mean, when, lit, create_map, regexp_extract
y_pred = model.predict(x_train,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_train, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TRAINING SET:
',classification_report(y_train,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))<train_model>
df1 = spark.read.csv('.. /input/titanic/train.csv',\
header=True, inferSchema=True)
 df2 = spark.read.csv('.. /input/titanic/test.csv', \
header=True, inferSchema=True )
print('fiting final model...')
 n_epochs = len(history.history['loss'])- patience
history = model.fit(x_train,y_train, batch_size=batch_size, epochs=n_epochs)
 print('fitting on full data done...' )<save_to_csv>
df1.limit(5 ).toPandas()
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_final=tokenizer.texts_to_sequences(df_final['question_text'])
 x_final = pad_sequences(x_final,maxlen=max_len)
 y_pred = model.predict(x_final,batch_size=batch_size,verbose=1)
 y_pred = y_pred > search_result['threshold']
y_pred = y_pred.astype(int)
 print(y_pred[:5])
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = y_pred
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<import_modules>
print('Number of rows: \t', df1.count())
 print('Number of columns: \t', len(df1.columns))
tqdm.pandas() 
<load_from_csv>
for col in df1.columns:
print(col.ljust(20), df1.filter(df1[col].isNull() ).count() )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )<feature_engineering>
df1 = df1.fillna({'Embarked': 'S', 'Fare':14.45} )
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))<string_transform>
df1 = age_imputer(df1, 'Mr', 33.02)
 df1 = age_imputer(df1, 'Mrs', 35.98)
 df1 = age_imputer(df1, 'Miss', 21.86)
 df1 = age_imputer(df1, 'Master', 4.75 )
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna("_na_" ).values
X_test = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]<set_options>
df1 = df1.withColumn('FamilySize', df1['Parch'] + df1['SibSp'] ).\
drop('Parch', 'SibSp' )
del train, test
gc.collect()<statistical_test>
df1 = df1.drop('PassengerID', 'Cabin', 'Name', 'Ticket', 'Title' )
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
for col in df1.columns:
print(col.ljust(20), df1.filter(df1[col].isNull() ).count() )
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )<statistical_test>
RandomForestClassifier, GBTClassifier
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<define_search_model>
stringIndex = StringIndexer(inputCols=['Sex', 'Embarked'],
outputCols=['SexNum', 'EmbNum'])
 stringIndex_model = stringIndex.fit(df1)
 df1_ = stringIndex_model.transform(df1 ).drop('Sex', 'Embarked')
 df1_.show(4 )
def capsule(inp):
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.24 )(x)
 x = Bidirectional(CuDNNLSTM(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=123000), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation='relu', kernel_initializer=glorot_normal(seed=123000))(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<compute_test_metric>
vec_asmbl = VectorAssembler(inputCols=df1_.columns[1:],
outputCol='features')
 df1_ = vec_asmbl.transform(df1_ ).select('features', 'Survived')
 df1_.show(4, truncate=False )
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2<train_model>
train_df, valid_df = df1_.randomSplit([0.7, 0.3] )
f1, threshold = f1_smart(np.squeeze(Y), oof)
 print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))<compute_test_metric>
evaluator = MulticlassClassificationEvaluator(labelCol='Survived',
metricName='accuracy' )
np.mean(bestscore), np.mean(logloss )<save_to_csv>
ridge = LogisticRegression(labelCol='Survived',
maxIter=100,
elasticNetParam=0,
regParam=0.03)
 model = ridge.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>threshold ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<load_pretrained>
lasso = LogisticRegression(labelCol='Survived',
maxIter=100,
elasticNetParam=1,
regParam=0.0003)
 model = lasso.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )
def save_model(model, model_path):

 torch.save(model.state_dict() , model_path)
 def load_model(model, model_path, use_cuda=False):

 map_location = 'cpu'
if use_cuda and torch.cuda.is_available() :
map_location = 'cuda:0'
model.load_state_dict(torch.load(model_path, map_location))
return model<init_hyperparams>
rf = RandomForestClassifier(labelCol='Survived',
numTrees=100, maxDepth=3)
 model = rf.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )
class TextCNN(nn.Module):

 def __init__(self, args):
super(TextCNN, self ).__init__()
vocab_size = args["vocab_size"]
pretrained_embed = args["pretrained_embed"]
padding_idx = args["padding_idx"]
num_classes = 1
kernel_nums = [100, 100, 100]
kernel_sizes = [3, 4, 5]
embed_dim = 300
hidden_dim = 100
drop_prob = 0.5
if pretrained_embed is None:
self.embed = nn.Embedding(vocab_size, embed_dim)
 else:
self.embed = nn.Embedding.from_pretrained(
 pretrained_embed, freeze=False)
 self.embed.padding_idx = padding_idx
self.convs = nn.ModuleList(
 [nn.Conv1d(embed_dim, kn, ks)
 for kn, ks in zip(kernel_nums, kernel_sizes)])
 self.fc = nn.Linear(sum(kernel_nums), hidden_dim)
 self.act = nn.ReLU()
self.drop = nn.Dropout(drop_prob)
 self.out = nn.Linear(hidden_dim, num_classes)
 self.loss = nn.BCEWithLogitsLoss()
def forward(self, word_seq):
e = self.drop(self.embed(word_seq))
e = e.transpose(1, 2)
 ps = []
for conv in self.convs:
c = conv(e)
 p = F.max_pool1d(c, kernel_size=c.size(-1)).squeeze(-1)
 ps.append(p)
 p = torch.cat(ps, dim=1)
 f = self.drop(self.act(self.fc(p)))
 logits = self.out(f ).squeeze(-1)
 return logits
class Trainer(object):

 def __init__(self, **kwargs):
self.n_epochs = kwargs["epochs"]
self.batch_size = kwargs["batch_size"]
self.validate = kwargs["validate"]
self.save_best_dev = kwargs["save_best_dev"]
self.use_cuda = kwargs["use_cuda"]
self.print_every_step = kwargs["print_every_step"]
self.optimizer = kwargs["optimizer"]
self.model_path = kwargs["model_path"]
self.eval_metrics = kwargs["eval_metrics"]
self._best_accuracy = 0.0
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def train(self, network, train_data, dev_data=None):
network = network.to(self.device)
 train_iter = torchtext.data.Iterator(
 dataset=train_data, batch_size=self.batch_size,
train=True, shuffle=True, sort=False,
device=self.device)
 if self.validate:
default_valid_args = {
"batch_size": max(8, self.batch_size // 10),
"use_cuda": self.use_cuda}
validator = Tester(**default_valid_args)
 start = time.time()
for epoch in range(1, self.n_epochs + 1):
network.train()
train_iter.init_epoch()
self._train_step(
 train_iter, network, start=start,
n_print=self.print_every_step, epoch=epoch)
 if self.validate:
if dev_data is None:
raise RuntimeError(
 "self.validate is True in trainer, "
"but dev_data is None."
" Please provide the validation data.")
 eval_results = validator.test(network, dev_data)
 if self.save_best_dev and self.best_eval_result(eval_results):
save_model(network, self.model_path)
 print("Saved better model selected by validation.")
 def _train_step(self, data_iterator, network, **kwargs):

 step = 0
for batch in data_iterator:
text, target = batch.text, batch.target
self.optimizer.zero_grad()
logits = network(text)
 loss = network.loss(logits, target.float())
 loss.backward()
self.optimizer.step()
if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0:
end = time.time()
diff = timedelta(seconds=round(end - kwargs["start"]))
print_output = "[epoch: {:>3} step: {:>4}]" \
" train loss: {:>4.6} time: {}".format(
 kwargs["epoch"], step, loss.item() , diff)
 print(print_output)
 step += 1
def best_eval_result(self, eval_results):

 assert self.eval_metrics in eval_results, \
"Evaluation doesn't contain metrics '{}'." \ 
.format(self.eval_metrics)
 accuracy = eval_results[self.eval_metrics]
if accuracy > self._best_accuracy:
self._best_accuracy = accuracy
return True
else:
return False
class Tester(object):

 def __init__(self, **kwargs):
self.batch_size = kwargs["batch_size"]
self.use_cuda = kwargs["use_cuda"]
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def test(self, network, dev_data, threshold=0.33):
network = network.to(self.device)
 network.eval()
output_list = []
truth_list = []
data_iter = torchtext.data.Iterator(
 dataset=dev_data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text, target = batch.text, batch.target
with torch.no_grad() :
prediction = network(text)
 output_list.append(prediction.detach())
 truth_list.append(target.detach())
 eval_results = self.evaluate(output_list, truth_list, threshold)
 print("[tester] {}".format(self.print_eval_results(eval_results)))
 return eval_results
def evaluate(self, predict, truth, threshold=0.33):

 y_trues, y_preds = [], []
for y_true, logit in zip(truth, predict):
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_true = y_true.cpu().numpy()
y_trues.append(y_true)
 y_preds.append(y_pred)
 y_true = np.concatenate(y_trues, axis=0)
 y_pred = np.concatenate(y_preds, axis=0)
 precision = metrics.precision_score(y_true, y_pred, pos_label=1)
 recall = metrics.recall_score(y_true, y_pred, pos_label=1)
 f1 = metrics.f1_score(y_true, y_pred, pos_label=1)
 metrics_dict = {"precision": precision, "recall": recall, "f1": f1}
return metrics_dict
def print_eval_results(self, results):

 return ", ".join(
 [str(key)+ "=" + "{:.4f}".format(value)
 for key, value in results.items() ])
 class Predictor(object):

 def __init__(self, batch_size=8, use_cuda=False):
self.batch_size = batch_size
self.use_cuda = use_cuda
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def predict(self, network, data, threshold=0.33):
network = network.to(self.device)
 network.eval()
batch_output = []
data_iter = torchtext.data.Iterator(
 dataset=data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text = batch.text
with torch.no_grad() :
prediction = network(text)
 batch_output.append(prediction.detach())
 return self._post_processor(batch_output, threshold)
 def _post_processor(self, batch_output, threshold=0.33):

 y_preds = []
for logit in batch_output:
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_preds.append(y_pred)
 y_pred = np.concatenate(y_preds, axis=0)
 return y_pred<define_variables>
gb = GBTClassifier(labelCol='Survived', maxIter=75, maxDepth=3)
 model = gb.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
embed_path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
submission_path = './submission.csv'
model_path = './default_model.pkl'<create_dataframe>
for col in df2.columns:
print(col.ljust(20), df2.filter(df2[col].isNull() ).count() )
def pre() :

 print("Pre-processing...")
 fix_length = 100
text = torchtext.data.Field(
 sequential=True, use_vocab=True, lower=True,
tokenize=nltk.word_tokenize, batch_first=True,
is_target=False, fix_length=fix_length)
 target = torchtext.data.Field(
 sequential=False, use_vocab=False,
batch_first=True, is_target=True)
 dataset = torchtext.data.TabularDataset(
 train_path, format='csv',
fields={"question_text":('text', text),
"target":('target', target)})
 data_test = torchtext.data.TabularDataset(
 test_path, format='csv',
fields={"question_text":('text', text)})
 text.build_vocab(dataset, data_test, min_freq=3)
 text.vocab.load_vectors(torchtext.vocab.Vectors(embed_path))
vocab_size = len(text.vocab.itos)
 padding_idx = text.vocab.stoi[text.pad_token]
data_train, data_val = dataset.split(split_ratio=0.9)
 print("train set size:", len(data_train))
print("val set size:", len(data_val))
print("test set size:", len(data_test))
print("vocab size:", len(text.vocab.itos))
print("embed shape:", text.vocab.vectors.shape)
 print('')
 args_dict = {
"data_train": data_train, "data_val": data_val,
"data_test": data_test, "vocab_size": vocab_size,
"padding_idx": padding_idx}
return args_dict<categorify>
df2 = df2.fillna({'Embarked': 'S', 'Fare':14.45})
 df2 = df2.withColumn('FamilySize', df2['Parch'] + df2['SibSp'] ).\
drop('Parch', 'SibSp' )
args = pre()<train_on_grid>
df2 = age_imputer(df2, 'Mr', 33.02)
 df2 = age_imputer(df2, 'Mrs', 35.98)
 df2 = age_imputer(df2, 'Miss', 21.86)
 df2 = age_imputer(df2, 'Master', 4.75)
 df2 = df2.drop('Cabin', 'Name', 'Ticket', 'Title')
 df2.show(4 )
def train(**args):

 print("Training...")
 data_train = args["data_train"]
pretrained_embed = data_train.fields["text"].vocab.vectors
model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": pretrained_embed,
}
model = TextCNN(model_args)
 trainer_args = {
"epochs": 8,
"batch_size": 128,
"validate": True,
"save_best_dev": True,
"use_cuda": True,
"print_every_step": 1000,
"optimizer": torch.optim.Adam(model.parameters() , lr=1e-3),
"model_path": model_path,
"eval_metrics": "f1",
}
trainer = Trainer(**trainer_args)
 data_val = args["data_val"]
trainer.train(model, data_train, dev_data=data_val)
 print('' )<train_model>
for col in df2.columns:
print(col.ljust(20), df2.filter(df2[col].isNull() ).count() )
train(**args )<train_on_grid>
pred_test = model_final.transform(df2)
 predictions = pred_test.select('PassengerId', 'prediction')
 predictions = predictions.\
withColumn('Survived', predictions['prediction'].\
cast('integer')).drop('prediction')
 predictions.show(5 )
def test(**args):

 print("Testing...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = TextCNN(model_args)
 load_model(model, model_path, use_cuda=True)
 tester_args = {
"batch_size": 128,
"use_cuda": True,
}
tester = Tester(**tester_args)
 data_val = args["data_val"]
best_thresh, best_f1 = 0., 0.
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 f1 = tester.test(model, data_val, threshold=thresh)["f1"]
print("threshold: {:>.2f} f1: {:>.4f}".format(thresh, f1))
if f1 > best_f1:
best_thresh, best_f1 = thresh, f1
args["threshold"] = best_thresh
print("best f1 on dev: {:>.4f} threshold: {:>.2f}".format(
 best_f1, best_thresh))
print('')
 return args<predict_on_test>
predictions.coalesce(1 ).write.csv('submission_file.csv', header=True )
def infer(**args):

 print("Predicting...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = TextCNN(model_args)
 load_model(model, model_path, use_cuda=True)
 predictor = Predictor(batch_size=128, use_cuda=False)
 data_test = args["data_test"]
threshold = args["threshold"]
y_pred = predictor.predict(model, data_test, threshold=threshold)
 test_df = pd.read_csv(test_path, index_col=False, header=0)
 data = {"qid": test_df["qid"], "prediction": y_pred}
subm_df = pd.DataFrame(data=data)
 subm_df.to_csv(submission_path, header=True, index=False)
 print("submission saved as {}.".format(submission_path))
print('' )<set_options>
spark.read.csv('submission_file.csv', header=True ).show(4 )
warnings.filterwarnings("ignore")
 all_files = glob.glob(".. /input/cellstack/*.csv")
 all_files<save_to_csv>
predictions.toPandas().to_csv('submission.csv', index=False )
<define_variables><EOS> 
model_final.write().save('titanic_classification.model' )