Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
warnings.filterwarnings('ignore' )
best_threshold = 0.01
best_score = 0.0
for threshold in range(1, 100):
threshold = threshold / 100
score = f1_score(y_val, val_y_pred > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
print("Score at threshold=0.5 is {}".format(f1_score(y_val, val_y_pred > 0.5)))
 print("Optimal threshold is {} with a score of {}".format(best_threshold, best_score))<save_to_csv>
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 train_set.shape, test_set.shape
y_pred =(y_pred > best_threshold ).astype(int)
 submission['prediction'] = y_pred
submission.to_csv('submission.csv', index=False )<set_options>
full_set = pd.concat([train_set, test_set])
 full_set.head()
start = time.time()
seed = 32
os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['OMP_NUM_THREADS'] = '4'
np.random.seed(seed)
 rn.seed(seed)
 session_conf = tf.ConfigProto(intra_op_parallelism_threads = 1,
inter_op_parallelism_threads = 1)
 tf.set_random_seed(seed)
 sess = tf.Session(graph = tf.get_default_graph() , config = session_conf)
 K.set_session(sess)
 train = pd.read_csv(".. /input/train.csv" ).fillna("missing")
 test = pd.read_csv(".. /input/test.csv" ).fillna("missing")
 embedding_file1 = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
embedding_file2 = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
embed_size = 300
max_features = 100000
max_len = 60<define_variables>
full_set['Age'][full_set['Age'].isnull() ] = full_set['Age'].median()
full_set['Age'] = full_set['Age'].astype(int )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&',
'/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '√', 'β', 'α', '∅', 'θ', '÷', '₹']
def clean_punct(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_punct(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_punct(x))<define_variables>
full_set['HasCabin'] = full_set['Cabin'].apply(lambda x: 0 if isinstance(x, float)else 1 )
sincere = train[train["target"] == 0]
insincere = train[train["target"] == 1]
print("Sincere questions {}; Insincere questions {}".format(sincere.shape[0], insincere.shape[0]))<compute_train_metric>
full_set['Embarked'][full_set['Embarked'].isnull() ] = 'S'
full_set['Embarked'] = full_set['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
def get_glove(embedding_file):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file))
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
return embeddings_index, emb_mean, emb_std
def get_para(embedding_file):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file,
encoding="utf8",
errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
return embeddings_index, emb_mean, emb_std
glove_index, glove_mean, glove_std = get_glove(embedding_file1)
 para_index, para_mean, para_std = get_para(embedding_file2 )<categorify>
full_set[ full_set['Fare'].isnull() ]
def get_embed(tokenizer = None, embeddings_index = None, emb_mean = None, emb_std = None):
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return nb_words, embedding_matrix<string_transform>
full_set['Fare'][ full_set['Fare'].isnull() ] = 0 .
tokenizer = Tokenizer(num_words = max_features, lower = True)
 tokenizer.fit_on_texts(train["question_text"])
 train_token = tokenizer.texts_to_sequences(train["question_text"])
 fake_test_token = tokenizer.texts_to_sequences(fake_test["question_text"])
 test_token = tokenizer.texts_to_sequences(test["question_text"])
 train_seq = pad_sequences(train_token, maxlen = max_len)
 fake_test_seq = pad_sequences(fake_test_token, maxlen = max_len)
 X_test = pad_sequences(test_token, maxlen = max_len)
 del train_token, fake_test_token, test_token; gc.collect()<load_pretrained>
full_set['Title'] = full_set['Name'].apply(get_title)
 print(full_set['Title'].unique() )
nb_words, embedding_matrix1 = get_embed(tokenizer = tokenizer, embeddings_index = glove_index,
emb_mean = glove_mean,
emb_std = glove_std)
 nb_words, embedding_matrix2 = get_embed(tokenizer = tokenizer, embeddings_index = para_index,
emb_mean = para_mean,
emb_std = para_std)
 embedding_matrix = np.mean([embedding_matrix1, embedding_matrix2], axis = 0)
 del embedding_matrix1, embedding_matrix2; gc.collect()
print("Embedding matrix completed!" )<choose_model_class>
commons = ['Mr','Mrs','Miss','Mme','Ms','Mlle']
rares = list(set(full_set['Title'].unique())- set(commons))
full_set['Title'] = full_set['Title'].replace('Ms','Miss')
 full_set['Title'] = full_set['Title'].replace('Mlle','Miss')
 full_set['Title'] = full_set['Title'].replace('Mme','Mrs')
 full_set['Title'][full_set['Title'].isin(rares)] = 'Rare'
full_set['Title'] = full_set['Title'].map({'Mr':0, 'Mrs':1, 'Miss':2, 'Rare':3} )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<find_best_params>
full_set['FamSize'] = full_set['Parch'] + full_set['SibSp'] + 1
def get_f1(true, val):
precision, recall, thresholds = precision_recall_curve(true, val)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_threshold = thresholds[np.argmax(F)]
return best_threshold, best_score<choose_model_class>
full_set['Sex'] = full_set['Sex'].map({'male':0, 'female':1} )
def build_model(units = 40, dr = 0.3):
inp = Input(shape =(max_len,))
embed_layer = Embedding(nb_words, embed_size, input_length = max_len,
weights = [embedding_matrix], trainable = False )(inp)
 x = SpatialDropout1D(dr, seed = seed )(embed_layer)
 x = Bidirectional(CuDNNLSTM(units, kernel_initializer = glorot_normal(seed = seed),
recurrent_initializer = orthogonal(gain = 1.0, seed = seed),
return_sequences = True))(x)
 x = Bidirectional(CuDNNGRU(units, kernel_initializer = glorot_normal(seed = seed),
recurrent_initializer = orthogonal(gain = 1.0, seed = seed),
return_sequences = True))(x)
 att = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 main = concatenate([att, avg_pool, max_pool])
 main = Dense(64, kernel_initializer = glorot_normal(seed = seed))(main)
 main = Activation("relu" )(main)
 main = Dropout(0.1, seed = seed )(main)
 out = Dense(1, activation = "sigmoid",
kernel_initializer = glorot_normal(seed = seed))(main)
 model = Model(inputs = inp, outputs = out)
 model.compile(loss = "binary_crossentropy",
optimizer = Adam() ,
metrics = None)
 return model<split>
full_set.drop(['Cabin','Name','Parch','PassengerId','SibSp','Ticket'], axis=1, inplace=True )
fold = 5
batch_size = 1024
epochs = 5
oof_pred = np.zeros(( train.shape[0], 1))
pred = np.zeros(( test_shape[0], 1))
fake_pred = np.zeros(( test_shape[0], 1))
thresholds = []
k_fold = StratifiedKFold(n_splits = fold, random_state = seed, shuffle = True)
 for i,(train_idx, val_idx)in enumerate(k_fold.split(train_seq, target)) :
print("-"*50)
 print("Trainging fold {}/{}".format(i+1, fold))
X_train, y_train = train_seq[train_idx], target[train_idx]
X_val, y_val = train_seq[val_idx], target[val_idx]
K.clear_session()
model = build_model(units = 60)
 model.fit(X_train, y_train, batch_size = batch_size, epochs = epochs,
validation_data =(X_val, y_val), verbose = 2)
 val_pred = model.predict(X_val, batch_size = batch_size)
 oof_pred[val_idx] = val_pred
fake_pred += model.predict(fake_test_seq, batch_size = batch_size)/fold
pred += model.predict(X_test, batch_size = batch_size)/fold
threshold, score = get_f1(y_val, val_pred)
 print("F1 score at threshold {} is {}".format(threshold, score))<compute_test_metric>
def fare_bin(fare):
if fare <= 7.8958:
return 0.
 elif 7.8958 < fare <= 14.4542:
return 1.
 elif 14.4542 < fare <= 31.2750:
return 2.
 else:
return 3 .
threshold, score = get_f1(target, oof_pred)
 print("F1 score after K fold at threshold {} is {}".format(threshold, score))
fake_test["pred"] =(fake_pred > threshold ).astype(int)
 print("Fake test F1 score is {}".format(f1_score(fake_test["target"], 
(fake_test["pred"] ).astype(int))))
test["prediction"] =(pred > threshold ).astype(int )<save_to_csv>
full_set['Fare'] = full_set['Fare'].apply(fare_bin ).astype(int )
submission = test[["qid", "prediction"]]
submission.to_csv("submission.csv", index = False)
 submission.head()<set_options>
full_set['Pclass'] = full_set['Pclass'].astype('str' )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(6017)
 print('Seeding done...' )<feature_engineering>
full_w_dum = pd.get_dummies(full_set)
 full_w_dum.head()
print('Preproccesing texts.... ')
 print('lower...')
 df["question_text"] = df["question_text"].apply(lambda x: x.lower())
 df_final["question_text"] = df_final["question_text"].apply(lambda x: x.lower())
 contraction_mapping = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have" }
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
print('contractions...')
 df["question_text"] = df["question_text"].apply(lambda x: clean_contractions(x,contraction_mapping))
df_final["question_text"] = df_final["question_text"].apply(lambda x: clean_contractions(x,contraction_mapping))
punct_mapping = {"‘": "'",
"₹": "e",
"´": "'",
"°": "",
"€": "e",
"™": "tm",
"√": " sqrt ",
"×": "x",
"²": "2",
"—": "-",
"–": "-",
"’": "'",
"_": "-",
"`": "'",
'“': '"',
'”': '"',
'“': '"',
"£": "e",
'∞': 'infinity',
'θ': 'theta',
'÷': '/',
'α': 'alpha',
'•': '.',
'à': 'a',
'−': '-',
'β': 'beta',
'∅': '',
'³': '3',
'π': 'pi',
}
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
print('clean special chars...')
 df["question_text"] = df["question_text"].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_final["question_text"] = df_final["question_text"].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
mispell_dict = {'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization'}
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
print('clean misspellings...')
 df["question_text"] = df["question_text"].apply(lambda x: correct_spelling(x,mispell_dict))
df_final["question_text"] = df_final["question_text"].apply(lambda x: correct_spelling(x,mispell_dict))<string_transform>
full_set = full_w_dum
dim = 300
num_words = 95000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(list(df['question_text'])+list(df_final['question_text']))
print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape)
 x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )<compute_train_metric>
cols = list(set(full_set.columns)- set(['Survived']))
X_train, X_test = full_set[:train_set.shape[0]][cols], full_set[train_set.shape[0]:][cols]
y_train = full_set[:train_set.shape[0]]['Survived']
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
models = [ LogisticRegression, SVC, LinearSVC, RandomForestClassifier, KNeighborsClassifier, XGBClassifier ]
mscores = []
lscores = ['f1','accuracy','recall','roc_auc']
np.random.seed(42)
 for elem in models:
mscores2 = []
model = elem()
for sc in lscores:
scores = cross_val_score(model, X_train, y_train, scoring=sc)
 mscores2.append(np.mean(scores))
mscores.append(mscores2 )
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_para.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<train_model>
order = np.argsort(np.mean(np.array(mscores), axis=1))
print(order )
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss<define_variables>
from sklearn.model_selection import StratifiedKFold, KFold
class CyclicLR(object):

 def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, torch.optim.Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
def chaos_lr(r):
lr_lambda = lambda iters: rel_val(iters)
 def rel_val(iteration):
x = 0.99
for i in range(iteration):
x = x*r*(1.-x)
 return x
return lr_lambda
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
best threshold is % f with score %f'%(best_threshold,best_score))
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 X_train1,X_val,y_train1,y_val = train_test_split(x_train, y_train)
 X_train2,X_test,y_train2,y_test = train_test_split(X_train1, y_train1)
 x_train_tensor = torch.tensor(X_train2, dtype=torch.long ).cuda()
y_train_tensor = torch.tensor(y_train2, dtype=torch.float32 ).cuda()
train_dataset = torch.utils.data.TensorDataset(x_train_tensor,y_train_tensor)
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
 x_val_tensor = torch.tensor(X_val, dtype=torch.long ).cuda()
y_val_tensor = torch.tensor(y_val, dtype=torch.float32 ).cuda()
val_dataset = torch.utils.data.TensorDataset(x_val_tensor,y_val_tensor)
 val_loader = torch.utils.data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=False)
 class Attention(nn.Module):
def __init__(self,hidden_lstm_size):
super(Attention,self ).__init__()
self.hidden_lstm_size = hidden_lstm_size
self.seq_len = max_len
weights = torch.zeros(self.hidden_lstm_size,1)
 nn.init.xavier_uniform_(weights)
 self.weights = nn.Parameter(weights)
 def forward(self,x):
x_c = x.contiguous().view(-1, self.hidden_lstm_size)
 a = torch.mm(x_c, self.weights)
 a = a.view(-1, self.seq_len)
 a = torch.tanh(a)
 s = F.softmax(a,dim=1)
 s = torch.unsqueeze(s, -1)
 weighted_input =s*x
return torch.sum(weighted_input, 1)
 class Sentiment(nn.Module):
def __init__(self,matrix_glove,matrix_para,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',matrix_glove.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.lin_dim = 64
self.n_layers = 1
self.embedding_glove = nn.Embedding(matrix_glove.shape[0],matrix_glove.shape[1])
 self.embedding_glove.weight = nn.Parameter(torch.tensor(matrix_glove, dtype=torch.float32))
self.embedding_glove.weight.requires_grad = False
self.embedding_para = nn.Embedding(matrix_para.shape[0],matrix_para.shape[1])
 self.embedding_para.weight = nn.Parameter(torch.tensor(matrix_para, dtype=torch.float32))
self.embedding_para.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=matrix_glove.shape[1]*2,
hidden_size=self.hidden_dim,
num_layers=self.n_layers,
bidirectional=True,
batch_first=True,
dropout=0.1)
 self.att_lstm = Attention(self.hidden_dim*2)
 self.linear1 = nn.Linear(2*self.hidden_dim,self.lin_dim)
 self.relu = nn.ReLU()
self.linear2 = nn.Linear(self.lin_dim,1)
 self.dropout = nn.Dropout(0.1)
 def forward(self,x):
hidden =(torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 hidden_gru = torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda()
e_glove = self.embedding_glove(x)
 e_para = self.embedding_para(x)
 e = torch.cat([e_glove,e_para],dim=-1)
 lstm_out, hidden = self.lstm(e, hidden)
 out_lstm = self.att_lstm(lstm_out)
 out = self.linear1(out_lstm)
 out = self.relu(out)
 out = self.dropout(out)
 return self.linear2(out)
 model = Sentiment(embedding_matrix_glov,embedding_matrix_para,batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.RMSprop(model.parameters() ,lr=1e-3)
 scheduler = CyclicLR(optimizer,base_lr=1e-3, max_lr=4e-3,
step_size=300., mode='exp_range',
gamma=0.99994)
 losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
val_list = list(val_loader)
 for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
for batch,(x_batch,y_true)in enumerate(list(iter(train_loader)) ,1):
model.train()
optimizer.zero_grad()
y_pred = model(x_batch ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
scheduler.batch_step()
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,(x_val_batch,y_true)in enumerate(val_list,1):
y_pred = model(x_val_batch ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )<choose_model_class>
results_kfold = []
for K in [5,6,7,8,9,10]:
model = XGBClassifier()
kfold = KFold(n_splits=K, random_state=42)
 res = cross_val_score(model, X_train, y_train, cv=kfold)
 results_kfold.append(( K,res.mean() *100, res.std() *100))
print(os.listdir())
 model = Sentiment(embedding_matrix_glov,embedding_matrix_para,batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))<prepare_output>
list(map(lambda x: print('Iteration nº {:2}, with acc.{:.12} and std.dev.{:.12}'.format(*x)) ,results_kfold)) ;
print('Threshold:',search_result['threshold'])
 print(x_test.shape)
 submission_dataset = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long ).cuda())
 submission_loader = torch.utils.data.DataLoader(dataset=submission_dataset,batch_size=batch_size, shuffle=False)
 pred = []
with torch.no_grad() :
for x in list(submission_loader):
model.eval()
pred += torch.sigmoid(model(x[0] ).squeeze(1)).cpu().data.numpy().tolist()
pred = np.array(pred)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<set_options>
results_strat_kfold = []
for K in [5,6,7,8,9,10]:
model = XGBClassifier()
kfold = StratifiedKFold(n_splits=K, random_state=42)
 res = cross_val_score(model, X_train, y_train, cv=kfold)
 results_strat_kfold.append(( K,res.mean() *100, res.std() *100))
tqdm.pandas()
warnings.filterwarnings("ignore", message="F-score is ill-defined and being set to 0.0 due to no predicted samples.")
 %matplotlib inline<load_from_csv>
list(map(lambda x: print('Iteration nº {:2}, with acc.{:.12} and std.dev.{:.12}'.format(*x)) ,results_strat_kfold)) ;
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print('Train data dimension: ', train_df.shape)
 display(train_df.head())
 print('Test data dimension: ', test_df.shape)
 display(test_df.head() )<create_dataframe>
xg_scores = []
kfold = StratifiedKFold(n_splits=7, random_state=42)
 for lamb in [.05,.1,.2,.3,.4,.5,.6 ]:
for eta in [.2,.19,.17,.15,.13,.11]:
model = XGBClassifier(learning_rate=eta, reg_lambda=lamb)
 res = cross_val_score(model, X_train, y_train, cv=kfold)
 xg_scores.append({'lamb':lamb, 'eta':eta, 'acc':res.mean() } )
enable_local_test = True
if enable_local_test:
n_test = len(test_df)
 train_df, local_test_df =(train_df.iloc[:-n_test].reset_index(drop=True),
train_df.iloc[-n_test:].reset_index(drop=True))
else:
local_test_df = pd.DataFrame([[None, None, 0], [None, None, 0]], columns=['qid', 'question_text', 'target'])
 n_test = 2<set_options>
sorted(xg_scores,key=lambda x: x['acc'], reverse=True)[0]
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<compute_test_metric>
model = XGBClassifier(learning_rate=.17, reg_lambda=.5)
 model.fit(X_train, y_train)
 predicted = model.predict(X_test )
<compute_test_metric><EOS> 
test_set['Survived'] = predicted.astype(int)
 test_set[['PassengerId','Survived']].to_csv('submission.csv', sep=',', index=False )