Kaggle competition: Heart Disease Prediction
<SOS> metric: categorizationaccuracy Kaggle data source: heart-disease-prediction<predict_on_test>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
df_train = pd.read_csv('/kaggle/input/dataanalyticscoaching/train.csv')
 df_test = pd.read_csv('/kaggle/input/dataanalyticscoaching/test.csv' )
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
df_train.isnull().sum().sum()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
df_test.isnull().sum().sum()
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
df_train['target'].value_counts()
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
cat_feats = ['sex', 'cp', 'fbs', 'restecg', 'exang', 'slope', 'ca', 'thal']
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
num_feats = list(set(df_train.columns)- set(cat_feats)- {'id', 'target'})
 print(num_feats )
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
X_train = df_train[cat_feats + num_feats]
y_train = df_train['target']
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<compute_test_metric>
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler
pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<save_to_csv>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names].values
pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<define_variables>
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_feats)) , 
('std_scaler', StandardScaler())
 ] )
MAX_SEQUENCE_LENGTH = 60
MAX_WORDS = 45000
EMBEDDINGS_LOADED_DIMENSIONS = 300<load_from_csv>
cat_pipeline = Pipeline([ 
('selector', DataFrameSelector(cat_feats)) ,
] )
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )<define_variables>
full_pipeline = FeatureUnion(transformer_list=[ 
('num_pipeline', num_pipeline), 
('cat_pipeline', cat_pipeline)
 ] )
BATCH_SIZE = 256
Q_FRACTION = 1
questions = df_train.sample(frac=Q_FRACTION)
 question_texts = questions["question_text"].values
question_targets = questions["target"].values
test_texts = df_test["question_text"].fillna("_na_" ).values
print(f"Working on {len(questions)} questions" )<load_pretrained>
X_prep_train = full_pipeline.fit_transform(X_train )
def load_embeddings(file):
embeddings = {}
with open(file, encoding="utf8", errors='ignore')as f:
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings = dict(get_coefs(*line.split(" ")) for line in f)
 print('Found %s word vectors.' % len(embeddings))
return embeddings<train_model>
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
tokenizer = Tokenizer(num_words=MAX_WORDS)
 tokenizer.fit_on_texts(list(df_train["question_text"].values))<categorify>
from sklearn.metrics import accuracy_score, classification_report
def create_embedding_weights(tokenizer, embeddings, dimensions):
not_embedded = defaultdict(int)
 word_index = tokenizer.word_index
words_count = min(len(word_index), MAX_WORDS)
 embeddings_matrix = np.zeros(( words_count, dimensions))
for word, i in word_index.items() :
if i >= MAX_WORDS:
continue
if word not in embeddings:
continue
embedding_vector = embeddings[word]
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
return embeddings_matrix<compute_train_metric>
LR_clf = LogisticRegression()
LR_clf.fit(X_prep_train, y_train)
 y_pred = LR_clf.predict(X_prep_train)
 print(classification_report(y_train, y_pred))
THRESHOLD = 0.35
class EpochMetricsCallback(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.f1s = []
self.precisions = []
self.recalls = []
def on_epoch_end(self, epoch, logs={}):
predictions = self.model.predict(self.validation_data[0])
 predictions =(predictions > THRESHOLD ).astype(int)
 predictions = np.asarray(predictions)
 targets = self.validation_data[1]
f1 = metrics.f1_score(targets, predictions)
 precision = metrics.precision_score(targets, predictions)
 recall = metrics.recall_score(targets, predictions)
 print(" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}" 
.format(f1, precision, recall))
self.f1s.append(f1)
 self.precisions.append(precision)
 self.recalls.append(recall)
 return
def display_model_history(history):
data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})
 ax = sns.lineplot(data=data, palette="pastel", linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')
 ax.xaxis.set_major_locator(MaxNLocator(integer=True))
sns.despine()
plt.show()
def display_model_epoch_metrics(epoch_callback):
data = pd.DataFrame(data = {
'F1': epoch_callback.f1s,
'Precision': epoch_callback.precisions,
'Recall': epoch_callback.recalls})
 ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', title='Epoch metrics')
 ax.xaxis.set_major_locator(MaxNLocator(integer=True))
sns.despine()
plt.show() 
<prepare_x_and_y>
from sklearn.model_selection import cross_val_score, KFold
X = pad_sequences(tokenizer.texts_to_sequences(question_texts), maxlen=MAX_SEQUENCE_LENGTH)
 Y = question_targets
test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH )<import_modules>
kfold = KFold(n_splits=5 )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
scores = cross_val_score(LR_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 print('Scores:',scores)
 print('Mean:',np.mean(scores))
print('Std:',np.std(scores))
def make_model(emb_weights):
tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name="tokenized_input")
 embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,
weights=[emb_weights],
trainable=False )(tokenized_input)
 embedding = SpatialDropout1D(0.15 )(embedding)
 lstm = Bidirectional(CuDNNGRU(64, return_sequences=True))(embedding)
 lstm = SpatialDropout1D(0.15 )(lstm)
 lstm = Bidirectional(CuDNNGRU(32, return_sequences=True))(lstm)
 a = Attention(MAX_SEQUENCE_LENGTH )(lstm)
 d1 = Dense(32 )(a)
 d1 = Dropout(0.15 )(d1)
 b = BatchNormalization()(d1)
 out = Dense(1, activation='sigmoid' )(b)
 model = Model(inputs=[tokenized_input], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
return model<split>
from sklearn.model_selection import GridSearchCV
train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)
 test_predictions = []
kaggle_predictions = []
embedding_files = [
".. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin",
".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt",
".. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec",
".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"]
load_embedding_functions = [
lambda: load_embeddings(embedding_files[1]),
lambda: load_embeddings(embedding_files[2]),
lambda: load_embeddings(embedding_files[3])]
for index, load_embeddings_fn in enumerate(load_embedding_functions):
print(f"Training {embedding_files[index]}")
 pretrained_embeddings = load_embeddings_fn()
pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)
 epoch_callback = EpochMetricsCallback()
model = make_model(pretrained_emb_weights)
 history = model.fit(x=train_X, y=train_Y, validation_split=0.015,
batch_size=BATCH_SIZE, epochs=5, verbose=2,
callbacks=[epoch_callback])
 display_model_history(history)
 display_model_epoch_metrics(epoch_callback)
 kaggle_predictions.append(model.predict([test_word_tokens], batch_size=1024, verbose=2))
test_predictions.append(model.predict([test_X]))<save_to_csv>
from sklearn.model_selection import GridSearchCV
avg = np.average(kaggle_predictions, axis=0)
 df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(avg > THRESHOLD ).astype(int)
 df_out.to_csv("submission.csv", index=False )<compute_test_metric>
param_grid = {
'C':[0.01, 0.03, 0.1, 0.3, 1, 3, 10],
'max_iter':[50, 100, 150],
}
LR_clf = LogisticRegression()
grid_search = GridSearchCV(LR_clf, param_grid, cv=kfold, scoring='accuracy', verbose=1)
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 LR_clf = grid_search.best_estimator_
scores = cross_val_score(LR_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 LR_clf.fit(X_prep_train, y_train)
 y_pred = LR_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 df_comp = pd.DataFrame([['Logistic Regression', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp
avg = np.average(test_predictions, axis=0)
 f1s = []
precisions = []
recalls = []
Ts = [x * 0.01 for x in range(20, 60)]
for t in Ts:
pred =(avg > t ).astype(int)
 f1s.append(metrics.f1_score(test_Y, pred))
precisions.append(metrics.precision_score(test_Y, pred))
recalls.append(metrics.recall_score(test_Y, pred))
data = pd.DataFrame(data = {'F1': f1s,
'Precision': precisions,
'Recall': recalls},
index=Ts)
 ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 ax.set(xlabel='Threshold', ylabel='Value', title='Threshold levels')
 sns.despine()
plt.show()
thresh = Ts[np.argmax(f1s)]
pred =(avg > thresh ).astype(int)
 f1 = metrics.f1_score(test_Y, pred)
 print("Test F1 {0:.4f} at threshold {1:.3f}".format(f1, thresh))
thresh = THRESHOLD
pred =(avg > thresh ).astype(int)
 f1 = metrics.f1_score(test_Y, pred)
 print("Test F1 {0:.4f} at threshold {1:.3f}".format(f1, thresh))<save_to_csv>
param_grid = {
'n_neighbors': range(1, 25, 5),
'weights':['uniform', 'distance'],
'algorithm':['auto', 'ball_tree', 'kd_tree', 'brute'],
}
KNN_clf = KNeighborsClassifier()
grid_search = GridSearchCV(KNN_clf, param_grid, cv=kfold, scoring='accuracy', verbose=1)
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 KNN_clf = grid_search.best_estimator_
scores = cross_val_score(KNN_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 KNN_clf.fit(X_prep_train, y_train)
 y_pred = KNN_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 temp = pd.DataFrame([['KNN', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp = df_comp.append(temp,ignore_index=True)
 df_comp
thresh = Ts[np.argmax(f1s)]
avg = np.average(kaggle_predictions, axis=0)
 df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(avg > thresh ).astype(int)
 df_out.to_csv("submission.csv", index=False )<import_modules>
param_grid = {
}
NB_clf = GaussianNB()
grid_search = GridSearchCV(NB_clf, param_grid, cv=kfold, scoring='accuracy')
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 NB_clf = grid_search.best_estimator_
scores = cross_val_score(NB_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 NB_clf.fit(X_prep_train, y_train)
 y_pred = NB_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 temp = pd.DataFrame([['Naive Bayes', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp = df_comp.append(temp,ignore_index=True)
 df_comp
import pandas as pd
import numpy as np
import torch
from torch import nn
from sklearn.metrics import f1_score
import torchtext
from tqdm import tqdm, tqdm_notebook
from nltk import word_tokenize
import random
from torch import optim<load_from_csv>
param_grid = {
'criterion':['gini', 'entropy'],
'splitter':['best','random'],
'max_depth':[None] + list(range(1,20, 2)) ,
'max_features':[None, 'auto', 'sqrt', 'log2'],
'ccp_alpha':[0.0, 0.01, 0.03, 0.1]
}
DT_clf = DecisionTreeClassifier()
grid_search = GridSearchCV(DT_clf, param_grid, cv=kfold, scoring='accuracy', verbose=1)
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 DT_clf = grid_search.best_estimator_
scores = cross_val_score(DT_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 DT_clf.fit(X_prep_train, y_train)
 y_pred = DT_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 temp = pd.DataFrame([['Decision Tree', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp = df_comp.append(temp,ignore_index=True)
 df_comp
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=70)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 test = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)} )<compute_train_metric>
param_grid = {
'C':[0.01, 0.03, 0.1, 0.3, 1, 3, 10],
'kernel':['linear', 'poly', 'rbf', 'sigmoid'],
'degree':list(range(1,5)) ,
}
SVM_clf = SVC()
grid_search = GridSearchCV(SVM_clf, param_grid, cv=kfold, scoring='accuracy', verbose=1)
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 SVM_clf = grid_search.best_estimator_
scores = cross_val_score(SVM_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 SVM_clf.fit(X_prep_train, y_train)
 y_pred = SVM_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 temp = pd.DataFrame([['SVM', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp = df_comp.append(temp,ignore_index=True)
 df_comp
text.build_vocab(train, test, min_freq=3)
 qid.build_vocab(test )<load_from_csv>
param_grid = {
'n_estimators':range(50,200,50),
'criterion':['gini', 'entropy'],
'bootstrap':[True, False],
'max_features':['auto', 'sqrt', 'log2'],
'ccp_alpha':[0.01, 0.03, 0.1]
}
RF_clf = RandomForestClassifier()
grid_search = GridSearchCV(RF_clf, param_grid, cv=kfold, scoring='accuracy', verbose=1, n_jobs=-1)
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 RF_clf = grid_search.best_estimator_
scores = cross_val_score(RF_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 RF_clf.fit(X_prep_train, y_train)
 y_pred = RF_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 temp = pd.DataFrame([['Random Forest', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp = df_comp.append(temp,ignore_index=True)
 df_comp
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 tqdm_notebook().pandas()<set_options>
param_grid = {
'loss':['deviance', 'exponential'],
'learning_rate':[0.01, 0.03, 0.1, 0.3, 1],
'n_estimators':list(range(50,200,50)) ,
'criterion':['friedman_mse', 'mse', 'mae'],
'ccp_alpha':[0.01, 0.03, 0.1]
}
GB_clf = GradientBoostingClassifier()
grid_search = GridSearchCV(GB_clf, param_grid, cv=kfold, scoring='accuracy', verbose=1, n_jobs=-1)
 grid_search.fit(X_prep_train, y_train)
 print(grid_search.best_params_)
 GB_clf = grid_search.best_estimator_
scores = cross_val_score(GB_clf, X_prep_train, y_train, scoring='accuracy', cv=kfold)
 GB_clf.fit(X_prep_train, y_train)
 y_pred = GB_clf.predict(X_prep_train)
 acc = accuracy_score(y_train, y_pred)
 temp = pd.DataFrame([['Grad.Boost.', np.round(acc,2), np.round(np.mean(scores),2)]],
columns=['Algo', 'Train_acc', 'Val_acc'])
 df_comp = df_comp.append(temp,ignore_index=True)
 df_comp
text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300 )<choose_model_class>
X_test = df_test[cat_feats + num_feats]
X_prep_test = full_pipeline.fit_transform(X_test )
class TextCNN(nn.Module):
def __init__(self, lm, padding_idx, static=True, kernel_num=128, fixed_length=50, kernel_size=[2, 5, 10], dropout=0.2):
super(TextCNN, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 self.embedding = nn.Embedding.from_pretrained(lm)
 if static:
self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.conv = nn.ModuleList([nn.Conv2d(1, kernel_num,(i, self.embedding.embedding_dim)) for i in kernel_size])
 self.maxpools = [nn.MaxPool2d(( fixed_length+1-i,1)) for i in kernel_size]
self.fc = nn.Linear(len(kernel_size)*kernel_num, 1)
 def forward(self, input):
x = self.embedding(input ).unsqueeze(1)
 x = [self.maxpools[i](torch.tanh(cov(x)) ).squeeze(3 ).squeeze(2)for i, cov in enumerate(self.conv)]
x = torch.cat(x, dim=1)
 y = self.fc(self.dropout(x))
return y<train_model>
y_pred_test = SVM_clf.predict(X_prep_test )
def search_best_f1(true, pred):
tmp = [0,0,0]
delta = 0
for tmp[0] in np.arange(0.1, 0.501, 0.01):
tmp[1] = f1_score(true, np.array(pred)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
return tmp[2], delta
def training(epoch, model, loss_func, optimizer, train_iter):
e = 0
while e < epoch:
train_iter.init_epoch()
losses, preds, true = [], [], []
for train_batch in tqdm(list(iter(train_iter)) , 'epcoh {} training'.format(e)) :
model.train()
x = train_batch.text.cuda()
y = train_batch.target.type(torch.Tensor ).cuda()
true.append(train_batch.target.numpy())
 model.zero_grad()
pred = model.forward(x ).view(-1)
 loss = loss_function(pred, y)
 preds.append(torch.sigmoid(pred ).cpu().data.numpy())
 losses.append(loss.cpu().data.numpy())
 loss.backward()
optimizer.step()
train_f1, alpha_train = search_best_f1([j for i in true for j in i], [j for i in preds for j in i])
 print('epcoh {:02} - train_loss {:.4f} - train f1 {:.4f} - delta {:.4f}'.format(
 e, np.mean(losses), train_f1, alpha_train))
e += 1
return alpha_train 
<define_variables>
submission = pd.DataFrame({
"id": df_test['id'],
"target": y_pred_test
})
 submission.to_csv('submission.csv', index=False )