Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<predict_on_test>
warnings.filterwarnings("ignore" )
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<compute_test_metric>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
pred_val_y = 0.39*pred_glove_val_y + 0.41*pred_fasttext_val_y + 0.39*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<save_to_csv>
combine = pd.concat([train,test],sort = True,ignore_index=True )
pred_test_y = 0.35*pred_glove_test_y + 0.42*pred_fasttext_test_y + 0.35*pred_paragram_test_y
pred_test_y =(pred_test_y>0.42 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
combine["Surname"] = combine["Name"].str.split(",",expand = True)[0]
combine["Surname"] = combine["Surname"].str.split("-",expand = True)[0]
combine[["Name","Surname"]].head(2 )
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split<load_from_csv>
for name in combine.Surname.values:
combine.loc[combine.Surname==name,"Surname_Counts"]=combine[combine.Surname==name].shape[0]
combine[["Surname","Surname_Counts"]].head(2 )
train_df = pd.read_csv(".. /input/train.csv")
 train_df, val_df = train_test_split(train_df, test_size=0.1 )<feature_engineering>
combine['Title'] = combine.Name.str.extract('([A-Za-z]+)\.', expand=False )
embeddings_index = {}
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))<prepare_x_and_y>
for i,j in combine.groupby(["Sex"]):
j_title_list = set(list(j.Title.values))
print(i,j_title_list )
def text_to_array(text):
empyt_emb = np.zeros(300)
 text = text[:-1].split() [:100]
embeds = [embeddings_index.get(x, empyt_emb)for x in text]
embeds+= [empyt_emb] *(100 - len(embeds))
return np.array(embeds)
 val_vects = np.array([text_to_array(X_text)for X_text in tqdm(val_df["question_text"][:5000])])
 val_y = np.array(val_df["target"][:5000])
<define_variables>
survival_1 = combine[(combine.Gender=="Boy")&(combine.Pclass.isin([1,2])) ]["Survived"].mean()
survival_2 = combine[(combine.Sex=="male")&(combine.Age<16)&(combine.Pclass.isin([1,2])) ]["Survived"].mean()
survival_3 = combine[(((combine.Sex=="male")&(combine.Age<16)) |(combine.Gender=="Boy")) &(combine.Pclass.isin([1,2])) ]["Survived"].mean()
print("The Survival probability of method1 is %.2f%%,the survival probability of method2 is %.2f%%,and the survival probability of method3 is %.2f%%"
%(survival_1*100,survival_2*100,survival_3*100))
batch_size = 128
def batch_gen(train_df):
n_batches = math.ceil(len(train_df)/ batch_size)
 while True:
train_df = train_df.sample(frac=1.)
for i in range(n_batches):
texts = train_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr, np.array(train_df["target"][i*batch_size:(i+1)*batch_size])
<import_modules>
combine.loc[(combine.Sex=="male")&(combine.Age<16),"Gender"]="Boy"
from keras.models import Sequential, Input
from keras.models import Model
from keras.layers import concatenate, GlobalAveragePooling1D, GlobalMaxPooling1D, CuDNNLSTM, CuDNNGRU, Dense, Bidirectional, SpatialDropout1D, Conv1D<define_search_model>
combine.drop(["Title","Sex"],axis = 1,inplace=True )
dr = 0.25
units = 96
inp = Input(shape =(( 100, 300)))
 x1 = SpatialDropout1D(dr )(inp)
 x = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 y = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
 y = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(y)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 avg_pool2 = GlobalAveragePooling1D()(y)
 max_pool2 = GlobalMaxPooling1D()(y)
 x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
 x = Dense(1, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] )<train_model>
combine_female = combine[combine.Gender == "Woman"]
combine_female.reset_index(inplace = True)
 for i,name in enumerate(list(combine_female.Name.values)) :
name = str(name)
 name = re.sub("\\s$","",name)
 name = re.sub(".*[^\\)]$","",name)
 name = re.sub(".*\\s (.*)\\)$","\\1",name)
 name = re.sub("^\\(","",name)
 combine_female.loc[i,"Maiden_name"] = name
combine_female[["Name","Maiden_name","Age"]].head(3 )
mg = batch_gen(train_df)
 model.fit_generator(mg, epochs=30,
steps_per_epoch=1000,
validation_data=(val_vects, val_y),
verbose=True )<define_variables>
for id in combine_female.PassengerId.values:
id2 = int(id)
 name = combine_female[combine_female.PassengerId==id]["Maiden_name"].values
combine.loc[combine.PassengerId==id2,"Maiden"] = name
combine[["Name","Maiden"]].head(2 )
batch_size = 256
def batch_gen(test_df):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr
test_df = pd.read_csv(".. /input/test.csv")
 all_preds = []
for x in tqdm(batch_gen(test_df)) :
all_preds.extend(model.predict(x ).flatten() )<predict_on_test>
combine.loc[combine.Maiden=="","Maiden"]=nan
all_preds_val = []
for x in tqdm(batch_gen(val_df)) :
all_preds_val.extend(model.predict(x ).flatten() )<prepare_x_and_y>
for mai in combine.Maiden.values:
combine.loc[combine.Maiden==mai,"Maiden_Counts"]=combine[combine.Maiden==mai].shape[0]
combine[["Maiden","Maiden_Counts"]].head(2 )
val_y = val_df["target"].values<import_modules>
combine["Family_size"] = combine["Parch"]+combine["SibSp"]+1
combine[["Name","Surname_Counts","Maiden_Counts","Family_size"]].head(2 )
from sklearn.metrics import f1_score<compute_test_metric>
pclass_gender_list = []
for pclass,gender in zip(combine.Pclass.values,combine.Gender.values):
pclass_gender = "P"+str(pclass)+"-"+str(gender)
 pclass_gender_list = pclass_gender_list+[pclass_gender]
y_val =(np.array(all_preds_val)> 0.1 ).astype(np.int)
 f1_score(val_y, y_val )<compute_test_metric>
combine["Pclass_Gender"] = pclass_gender_list
combine[["Pclass","Gender","Pclass_Gender"]].head(2 )
f1_score(val_y, y_val )<compute_test_metric>
combine["Ticket"].value_counts() [:5]
score = 0
thresh =.5
for i in np.arange(0.1, 0.991, 0.01):
y_val =(np.array(all_preds_val)> i ).astype(np.int)
 temp_score = f1_score(val_y, y_val)
 if(temp_score > score):
score = temp_score
thresh = i
print("CV: {}, Threshold: {}".format(score, thresh))<save_to_csv>
combine["Ticket_new"] = combine["Ticket"].str.extract('([0-9]+)$',expand = False )
y_te =(np.array(all_preds)> thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<import_modules>
combine[combine.Ticket_new.isna() ][["Ticket","Ticket_new"]]
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split<define_variables>
combine.loc[combine.Ticket_new.isna() ,"Ticket_new"]="LINE"
SEQ_LEN = 100<load_from_csv>
for i,tic in enumerate(combine.Ticket_new.values):
combine.loc[i,"Ticket"] = tic
combine.drop(["Ticket_new"],axis = 1,inplace = True )
train_df = pd.read_csv(".. /input/train.csv")
 train_df, val_df = train_test_split(train_df, test_size=0.07 )<feature_engineering>
combine["Ticket"].value_counts() [:5]
embeddings_index = {}
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))<string_transform>
tic_list = combine.Ticket.values
for i in tic_list:
combine.loc[combine["Ticket"]==i,"Ticket_Counts"] = combine[combine["Ticket"] == i].shape[0]
combine[["Ticket","Ticket_Counts"]].head(2 )
_WORD_SPLIT = re.compile("([.,!?"':; )(])")
 _DIGIT_RE = re.compile(br"\d")
 STOP_WORDS = "" ' [ ]., ! : ; ?".split(" ")
 def basic_tokenizer(sentence):

 words = []
for space_separated_fragment in sentence.strip().split() :
words.extend(_WORD_SPLIT.split(space_separated_fragment))
return [w.lower() for w in words if w != '' and w != ' ']<prepare_x_and_y>
for i,ti in enumerate(combine.Ticket.values):
t = list(ti)[:-2]+list("xx")
 d = ''
combine.loc[i,"Ticket2"] = d.join(t )
def text_to_array(text):
empyt_emb = np.zeros(300)
 text = basic_tokenizer(text[:-1])[:SEQ_LEN]
embeds = [embeddings_index.get(x, empyt_emb)for x in text]
embeds+= [empyt_emb] *(SEQ_LEN - len(embeds))
return np.array(embeds)
 val_vects = np.array([text_to_array(X_text)for X_text in tqdm(val_df["question_text"][:3000])])
 val_y = np.array(val_df["target"][:3000] )<define_variables>
tic2_list = combine.Ticket2.values
for i in tic2_list:
combine.loc[combine["Ticket2"]==i,"Ticket2_Counts"] = combine[combine["Ticket2"] == i].shape[0]
combine[["Ticket","Ticket2","Ticket2_Counts"]].head(2 )
batch_size = 256
def batch_gen(train_df):
n_batches = math.ceil(len(train_df)/ batch_size)
 while True:
train_df = train_df.sample(frac=1.)
for i in range(n_batches):
texts = train_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr, np.array(train_df["target"][i*batch_size:(i+1)*batch_size] )<import_modules>
have_cabin = combine[combine.Cabin.notna() ]
have_cabin.reset_index(inplace = True)
 multi_cabin = []
single_cabin = []
for i,j in enumerate(have_cabin.Cabin.values):
passid = have_cabin.loc[i].PassengerId
if len(j)>4:
multi_cabin = multi_cabin+[passid]
else:
single_cabin = single_cabin+[passid]
print(multi_cabin )
from keras.models import Sequential,Model
from keras.layers import CuDNNLSTM, Dense, Bidirectional, Input,Dropout
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints<choose_model_class>
cabin_title = have_cabin[have_cabin.PassengerId.isin(single_cabin)].Cabin.str.extract('^([A-Z])',expand=False)
 have_cabin.loc[have_cabin.PassengerId.isin(single_cabin),"Cabin_Title"] = cabin_title
have_cabin[["Cabin","Cabin_Title"]].head(2 )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
multi_cabin_title = []
for cabin in have_cabin[have_cabin.PassengerId.isin(multi_cabin)]["Cabin"].values:
cabinA = cabin.split(" ")[0]
cabinA_title = list(cabinA)[0]
cabinB = cabin.split(" ")[1]
cabinB_title = list(cabinB)[0]
if cabinA_title==cabinB_title:
multi_cabin_title = multi_cabin_title+[cabinA_title]
else:
multi_cabin_title = multi_cabin_title+[cabinA_title+cabinB_title]
have_cabin.loc[(have_cabin.PassengerId.isin(multi_cabin)) ,"Cabin_Title"] = multi_cabin_title
have_cabin[have_cabin.PassengerId.isin(multi_cabin)][["Cabin","Cabin_Title"]][:3]
inp = Input(shape=(SEQ_LEN,300))
x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64,return_sequences=True))(x)
 x = Attention(SEQ_LEN )(x)
 x = Dense(256, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] )<train_model>
cabin_title_list = list(have_cabin.Cabin_Title.values)
 combine.loc[(combine.Cabin.notna()),"Cabin_Title"] = cabin_title_list
combine[combine.Cabin.notna() ][["Cabin","Cabin_Title"]].head(2 )
mg = batch_gen(train_df)
 model.fit_generator(mg, epochs=20,
steps_per_epoch=1000,
validation_data=(val_vects, val_y),
verbose=True )<define_variables>
combine.loc[(combine.Fare.notna()),"Fare_PP"] = combine.Fare/combine.Ticket_Counts
combine[["Fare","Fare_PP","Ticket_Counts"]].sample(5 )
batch_size = 256
def batch_gen(test_df):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr
test_df = pd.read_csv(".. /input/test.csv")
 all_preds = []
for x in tqdm(batch_gen(test_df)) :
all_preds.extend(model.predict(x ).flatten() )<save_to_csv>
combine[combine.Fare.isna() ]
y_te =(np.array(all_preds)> 0.35 ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<import_modules>
combine.loc[(combine.Fare_PP.isna()),"Fare_PP"] = combine[(combine.Embarked=="S")&(combine.Pclass==3)]["Fare_PP"].median()
combine.loc[(combine.Fare.isna()),"Fare"] = combine[(combine.Embarked=="S")&(combine.Pclass==3)]["Fare_PP"].median()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
multi_ticket = list(combine[(combine.Ticket_Counts>=2)&(combine.Cabin_Title.notna())]["Ticket"].values)
 len(set(multi_ticket))
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
multi_ticket_cabin = combine[(combine.Cabin_Title.notna())&(combine.Ticket_Counts>=2)][["Ticket","Cabin_Title"]]
same_ticket_differ_cabin = []
for i,j in multi_ticket_cabin.groupby(["Ticket"]):
differ_cabin = set(list(j.Cabin_Title.values))
if len(differ_cabin)>=2:
same_ticket_differ_cabin = same_ticket_differ_cabin+[i]
print(same_ticket_differ_cabin )
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<choose_model_class>
need_filling = list(combine[(combine.Ticket.isin(multi_ticket)) &(combine.Cabin.isna())].PassengerId.values)
 for passid in need_filling:
ticket = list(combine[combine.PassengerId==passid]["Ticket"].values)
 cabin_title = combine[combine.Ticket.isin(ticket)]["Cabin_Title"].describe() ["top"]
combine.loc[combine.PassengerId==passid,"Cabin_Title"] = cabin_title
combine[combine.PassengerId.isin(need_filling)][["Cabin_Title"]][:3]
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
combine.drop(["Cabin","Fare"],axis = 1,inplace = True )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
need_check = combine[(combine.PassengerId.isin(maybe_wrong)) &(combine.Ticket_Counts==1)]["PassengerId"].values
need_check
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_noemb_val_y>thresh ).astype(int))))<predict_on_test>
def get_from_passid(dataset,passid_list):
return dataset[dataset.PassengerId.isin(passid_list)][['Age', 'Cabin_Title', 'Embarked', 'Name', 'Parch', 'SibSp','Ticket',
'Ticket2', 'Surname','Maiden',"Ticket_Counts", 'Pclass_Gender',
'Family_size',"Survived"]]
def get_from_name(dataset,name_list):
return dataset[(dataset.Surname.isin(name_list))
|(dataset.Maiden.isin(name_list)) ][['Age', 'Cabin_Title', 'Embarked', 'Name', 'Parch', 'SibSp','Ticket',
'Ticket2', 'Surname','Maiden',"Ticket_Counts", 'Pclass_Gender',
'Family_size',"Survived"]]
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
combine[(combine.Ticket2=="175xx")&(combine.Cabin_Title=="C")&(combine.Embarked=="C")&(combine.Family_size>=2)&(combine.Pclass==1)]
del model, inp, x
time.sleep(10 )<statistical_test>
combine[(( combine.Age<=3)|(combine.Age>=32))
&(combine.Embarked=="S")&(combine.Family_size>=2)
 &(combine.Ticket2=="2506xx")&(combine.Pclass==2)][['Age', 'Cabin_Title', 'Embarked', 'Name', 'Parch', 'SibSp','Ticket',
'Ticket2', 'Surname','Maiden',"Ticket_Counts", 'Pclass_Gender', 'Family_size']]
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
combine[(combine.Embarked=="S")&(combine.Ticket2=="3470xx")&(combine.Pclass==3)&(combine.Family_size==2)][['Age', 'Cabin_Title', 'Embarked', 'Name', 'Parch', 'SibSp','Ticket', 'Surname', 'Maiden',
'Pclass_Gender', 'Family_size', 'Surname_Counts', 'Ticket_Counts',"Maiden_Counts","Survived"]]
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
combine.loc[combine.PassengerId.isin([137]),"Maiden"] = "Monypeny"
combine.loc[combine.Maiden.isin(["Monypeny"]),"Maiden_Counts"] = 2
combine.loc[combine.PassengerId.isin([274]),"Parch"] = 0
combine.loc[combine.PassengerId.isin([274]),"Family_size"] = 1
combine.loc[combine.PassengerId.isin([418]),"Parch"] = 0
combine.loc[combine.PassengerId.isin([418]),"Family_size"] = 1
combine.loc[combine.PassengerId.isin([1130]),"Parch"] = 0
combine.loc[combine.PassengerId.isin([1130]),"SibSp"] = 0
combine.loc[combine.PassengerId.isin([1130]),"Family_size"] = 1
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
test_error = pd.DataFrame(columns=["Category","Train_data","Test_data","Error_Counts","Error_Probability"])
 def get_error_counts(pg,index):
PG_train_size = combine[(combine.Pclass_Gender.isin(pg)) &(combine.Survived.notna())].shape[0]
PG_test_size = combine[(combine.Pclass_Gender.isin(pg)) &(combine.Survived.isna())].shape[0]
PG_survival = combine[(combine.Pclass_Gender.isin(pg)) ]["Survived"].mean()
if PG_survival>=0.5:
error_probability = 1-PG_survival
try:
test_error_counts = round(error_probability*PG_test_size)
 except:
test_error_counts=0
else:
error_probability = PG_survival
test_error_counts = round(error_probability*PG_test_size)
 test_error.loc[index]=[pg,PG_train_size,PG_test_size,test_error_counts,"%.2f%%"%(round(PG_survival*100,2)) ]
index = 0
PG_list = [["P1-Woman","P2-Woman"],["P3-Woman"],["P1-Man"],["P2-Man"],["P3-Man"],["P1-Boy","P2-Boy"],["P3-Boy"]]
for pg in PG_list:
get_error_counts(pg,index)
 index +=1
test_error.sort_values(by = "Error_Counts",ascending = False )
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
combine.loc[(combine.Pclass_Gender.isin(["P1-Boy","P2-Boy","P1-Woman","P2-Woman"])) ,"Predict"] = 1
combine.loc[(combine.Pclass_Gender=="P2-Man"),"Predict"] = 0
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
have_family = combine[combine.Family_size>=2]
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
have_family[have_family.Ticket2_Counts==1][['Age','Cabin_Title', 'Embarked','Pclass_Gender','Name',
"Surname",'Maiden', 'Parch','SibSp', 'Family_size','Ticket',"Ticket_Counts"]]
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def get_team_id(dataset,name_list):
return dataset[(dataset.Surname.isin(name_list)) |(dataset.Maiden.isin(name_list)) ]["PassengerId"].values
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
team_id =get_team_id(have_family,["Olsen"])
 have_family.loc[have_family.PassengerId.isin(team_id),"Group_size"] = have_family[have_family.PassengerId.isin(team_id)].shape[0]
have_family.loc[have_family.PassengerId.isin(team_id),"Ave_Survival"] = have_family[have_family.PassengerId.isin(team_id)]["Survived"].mean()
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
team_id = get_team_id(have_family,["Crosby","Halstead"])
 have_family.loc[have_family.PassengerId.isin(team_id),"Group_size"] = have_family[have_family.PassengerId.isin(team_id)].shape[0]
have_family.loc[have_family.PassengerId.isin(team_id),"Ave_Survival"] = have_family[have_family.PassengerId.isin(team_id)]["Survived"].mean()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
def full_gs_and_as(dataset,id_list,ticket_kind,check_list,with_friend,count_maiden):
for i in id_list:
surname = set(dataset[dataset.PassengerId.isin([i])]["Surname"].values)
 ticket_list = set(dataset[dataset.PassengerId.isin([i])][ticket_kind].values)
 if count_maiden ==True:
maiden = set(dataset[dataset.PassengerId.isin([i])&(dataset.Maiden.notna())].Maiden.values)
 team = dataset[(( dataset.Surname.isin(surname|maiden))
|(dataset.Maiden.isin(surname|maiden)))
 &(dataset[ticket_kind].isin(ticket_list)) ]
team_pg = team.Pclass_Gender.describe().top
if ticket_kind=="Ticket":
team_size = team.shape[0]
ave_survival = dataset[dataset[ticket_kind].isin(ticket_list)].Survived.mean()
team_passid = list(team.PassengerId.values)
 else:
team_size = round(team.Family_size.describe().mean())
 ticket = set(team.Ticket.values)
 ave_survival = dataset[dataset.Ticket.isin(ticket)].Survived.mean()
team_passid = list(team.PassengerId.values)
 else:
team = dataset[(dataset[ticket_kind].isin(ticket_list)) &(dataset.Surname.isin(surname)) &(dataset.Group_size.isna())]
team_size = team.shape[0]
if team_size >=1:
team_pg = team.Pclass_Gender.describe().top
else:
team_pg = []
if team_size>=2:
ticket = set(team.Ticket.values)
 ticket_team_size = dataset[dataset.Ticket.isin(ticket)].shape[0]
if team_size == ticket_team_size:
team_passid = list(team.PassengerId.values)
 ave_survival=team.Survived.mean()
else:
team_passid = []
ave_survival = nan
with_friend = with_friend+[i]
else:
team_passid = []
ave_survival = nan
empty_survived = dataset[(dataset.PassengerId.isin(team_passid)) &(dataset.Survived.isna())].shape[0]
full_survived = dataset[(dataset.PassengerId.isin(team_passid)) &(dataset.Survived.notna())].shape[0]
if full_survived>0:
if empty_survived>0:
empty_gender_count = team[(team.Survived.isna())].Gender.describe() ["unique"]
empty_gender = set(team[(team.Survived.isna())].Gender.values)
 full_gender = set(team[(team.Survived.notna())].Gender.values)
 if ave_survival ==0:
if("Woman" in full_gender)|(( "Man" in empty_gender)&(empty_gender_count==1)) :
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 0
else:
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 0.5
elif ave_survival==1:
if("Man" in full_gender)|(( "Woman" in empty_gender)&(empty_gender_count==1)) :
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 1
else:
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 0.5
else:
check_list=check_list+[team_passid]
else:
dataset.loc[(dataset.PassengerId.isin(team_passid)) ,"Ave_Survival"]=ave_survival
else:
dataset.loc[(dataset.PassengerId.isin(team_passid)) ,"Ave_Survival"]=nan
dataset.loc[(dataset.PassengerId.isin(team_passid)) ,"Group_size"] = team_size
print("These passenger travel not only with cousin:",with_friend)
 check_list = list(np.unique(check_list))
print("these passenger need check more:",check_list )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
full_gs_and_as(dataset = have_family,id_list=ship_team_2,ticket_kind = "Ticket",check_list = [],with_friend=[],count_maiden = True )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def fillfull_p2(dataset,check_list,check_more):
for i in check_list:
pclass = dataset[dataset.PassengerId.isin(i)].Pclass.describe() ["mean"]
if pclass != 2:
check_more = check_more+[i]
else:
dataset.loc[dataset.PassengerId.isin(i),"Group_size"] = len(i)
 dataset.loc[dataset.PassengerId.isin(i),"Ave_Survival"] = dataset[dataset.PassengerId.isin(i)]["Survived"].mean()
return print(check_more )
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
check_list = [[26, 183, 234, 262, 1046, 1066, 1271], [28, 89, 342, 439, 945, 961], [44, 609, 686, 1188], [59, 451, 473, 1142],
[94, 789, 924, 1246], [119, 300, 1076], [246, 413, 1303], [280, 747, 1284], [298, 306, 499, 1198], [671, 685, 1067],
[818, 828, 1253]]
fillfull_p2(dataset = have_family,check_list = check_list,check_more = [] )
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
def check_ave_survival(dataset,passid):
return dataset[dataset.PassengerId.isin(passid)][["Pclass_Gender","Survived"]]
def fullfill_ave_survival(dataset,check_list,check_more):
for i in check_list:
ave_survival = dataset[dataset.PassengerId.isin(i)].Survived.mean()
empty_gender = list(dataset[(dataset.PassengerId.isin(i)) &(dataset.Survived.isna())].Gender.values)
 if ave_survival>0.5:
if "Man" in empty_gender:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = 0.5
else:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = ave_survival
elif ave_survival==0.5:
if "Woman"in empty_gender:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = 0.5
else:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = ave_survival
else:
check_more = check_more+[i]
return print(check_more )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<compute_test_metric>
check_ave_survival(have_family,[298, 306, 499, 1198] )
pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<save_to_csv>
have_family.loc[(have_family.PassengerId.isin([298, 306, 499, 1198])) ,"Ave_Survival"] = 0.33
pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<categorify>
situation_2 = ship_team_1+ship_team_2_tic2
full_gs_and_as(dataset = have_family,id_list=situation_2,ticket_kind = "Ticket2",check_list = [],with_friend=[],count_maiden = True )
pretrain_dir = None
one_fold = False
run_test = False
denoise = True
ae_epochs = 20
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [30, 10, 3, 3, 5, 5]
batch_size_list = [8, 16, 32, 64, 128, 256]
if pretrain_dir is not None:
for d in glob.glob(pretrain_dir + "*"):
shutil.copy(d, ".")
 %matplotlib inline<load_from_csv>
check_list = [[70, 185, 1057, 1268, 1286], [408, 438, 530, 775, 832, 944], [438, 530, 775, 944], [477, 727, 923, 1211],
[672, 821, 984, 1200], [727, 923, 1211], [821, 984, 1200]]
fillfull_p2(have_family,check_list,check_more = [] )
train = pd.read_json("/kaggle/input/stanford-covid-vaccine/train.json",lines=True)
 if denoise:
train = train[train.signal_to_noise > 1].reset_index(drop = True)
 test = pd.read_json("/kaggle/input/stanford-covid-vaccine/test.json",lines=True)
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
 if run_test:
train = train[:30]
test_pub = test_pub[:30]
test_pri = test_pri[:30]
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )<concatenate>
check_list = [[70, 185, 1057, 1268, 1286], [672, 821, 984, 1200], [821, 984, 1200]]
fullfill_ave_survival(have_family,check_list,check_more=[] )
targets = list(sub.columns[1:])
 print(targets)
 y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 y.shape<concatenate>
remain_second_family = list(have_family[(have_family.PassengerId.isin(have_secondary_family1+have_secondary_family2))
&(have_family.Group_size.isna())]["PassengerId"].values)
 print(len(remain_second_family))
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 print(Ss.shape)
 return Ss
Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )<compute_test_metric>
second_family = have_secondary_family1+have_secondary_family2
for i in remain_second_family:
second_family.remove(i)
 print(second_family )
def get_distance_matrix(As):
idx = np.arange(As.shape[1])
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1/Ds
Ds = Ds[None, :,:]
Ds = np.repeat(Ds, len(As), axis = 0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis = 3)
 print(Ds.shape)
 return Ds
Ds = get_distance_matrix(As)
 Ds_pub = get_distance_matrix(As_pub)
 Ds_pri = get_distance_matrix(As_pri )<data_type_conversions>
have_family[have_family.PassengerId.isin([70, 185, 1057, 1268, 1286])][["Pclass_Gender","Survived","Ave_Survival"]]
As = np.concatenate([As[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float32)
 As_pub = np.concatenate([As_pub[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float32)
 As_pri = np.concatenate([As_pri[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float32)
 del Ss, Ds, Ss_pub, Ds_pub, Ss_pri, Ds_pri
As.shape, As_pub.shape, As_pri.shape<categorify>
next_check_2 = []
have_grouped = []
for i,j in have_family[have_family.PassengerId.isin(next_check)].groupby(["Surname"]):
if(len(j)) >=2:
family_size = j.Family_size.mean()
j_size = j.shape[0]
j_passid = list(j.PassengerId.values)
 if j_size == family_size:
have_grouped = have_grouped+[j_passid]
have_family.loc[(have_family.PassengerId.isin(j_passid)) ,"Group_size"] = j_size
have_family.loc[(have_family.PassengerId.isin(j_passid)) ,"Ave_Survival"] = j.Survived.mean()
else:
next_check_2 = next_check_2+[j_passid]
else:
j_passid = list(j.PassengerId.values)
 next_check_2 = next_check_2+[j_passid]
print("have grouped",have_grouped)
 print("next more check",next_check_2 )
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = [".", "(", ")"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_structure = np.stack(train["structure"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_node = np.concatenate([X_node, X_loop], axis = 2)
 a = np.sum(X_node *(2 ** np.arange(X_node.shape[2])[None, None, :]), axis = 2)
 vocab = sorted(set(a.flatten()))
print(vocab)
 ohes = []
for v in vocab:
ohes.append(a == v)
 ohes = np.stack(ohes, axis = 2)
 X_node = np.concatenate([X_node, ohes], axis = 2 ).astype(np.float32)
 print(X_node.shape)
 return X_node
X_node = get_input(train)
 X_node_pub = get_input(test_pub)
 X_node_pri = get_input(test_pri )<compute_test_metric>
need_more_check = [41,193,69,1106,581,1133,1296,248,756,722,313,1041,443,893]
can_groupby_ticket2 = []
maybe_wrong = []
for i,j in have_family[have_family.PassengerId.isin(need_more_check)].groupby(["Embarked","Ticket2","Pclass"]):
if len(j)>=2:
can_groupby_ticket2 = can_groupby_ticket2+[i[1]]
else:
maybe_wrong = maybe_wrong+[i[1]]
print("can groupby ticket2",can_groupby_ticket2)
 print("maybe wrong",maybe_wrong )
def mcrmse(t, p, seq_len_target = seq_len_target):
score = np.mean(np.sqrt(np.mean(( p - y_va)** 2, axis = 2)) [:, :seq_len_target])
 return score
def mcrmse_loss(t, y, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
y = y[:, :seq_len_target]
loss = tf.reduce_mean(tf.sqrt(tf.reduce_mean(( t - y)** 2, axis = 2)))
 return loss
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 128, kernel = 3, rate = 0.0)
 x2 = forward(x1, 64, kernel = 6, rate = 0.0)
 x3 = forward(x2, 32, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [64, 32]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(X_node.shape[2], "sigmoid" )(x)
 loss = - tf.reduce_mean(20 * node * tf.math.log(p + 1e-4)+(1 - node)* tf.math.log(1 - p + 1e-4))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def get_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([node, adj])
 x = forward(x, 128, rate = 0.4)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam<train_model>
def get_from_id_tic2(dataset,passid,tic2):
return dataset[(dataset.PassengerId.isin(passid))
&(dataset.Ticket2.isin(tic2)) ][['Age', 'Embarked', 'Parch', 'SibSp','Ticket','Surname', 'Maiden','Family_size',
'Pclass_Gender','Cabin_Title',"Survived","Ave_Survival"]]
config = {}
if ae_epochs > 0:
base = get_base(config)
 ae_model = get_ae_model(base, config)
 for i in range(ae_epochs//ae_epochs_each):
print(f"------ {i} ------")
 print("--- train ---")
 ae_model.fit([X_node, As], [X_node[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- public ---")
 ae_model.fit([X_node_pub, As_pub], [X_node_pub[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- private ---")
 ae_model.fit([X_node_pri, As_pri], [X_node_pri[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 gc.collect()
print("****** save ae model ******")
 base.save_weights("./base_ae" )<define_variables>
get_from_id_tic2(have_family,need_more_check,["2377xx"] )
kfold = KFold(5, shuffle = True, random_state = 42)
 scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As)) :
print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("./base_ae")
 model = get_model(base, config)
 if pretrain_dir is not None:
d = f"./model{i}"
print(f"--- load from {d} ---")
 model.load_weights(d)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va, As_va], [y_va]),
epochs = epochs,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"./model{i}")
 p = model.predict([X_node_va, As_va])
 scores.append(mcrmse(y_va, p))
print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
if one_fold:
break
pd.to_pickle(preds, "oof.pkl" )<predict_on_test>
have_family.loc[(have_family.PassengerId.isin([581,1133])) ,["Group_size","Ave_Survival"]] = [2,1]
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
get_from_id_tic2(have_family,need_more_check,['2506xx'] )
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False)
 preds_df.head()<set_options>
have_family.loc[(have_family.PassengerId.isin([248,313,756,1041])) ,["Group_size","Ave_Survival"]] = [3,0.5]
warnings.filterwarnings('ignore' )<import_modules>
get_from_id_tic2(have_family,need_more_check,['3470xx'] )
import xgboost as xgb<categorify>
get_from_id_tic2(have_family,need_more_check,['3500xx'] )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]<define_variables>
have_family.loc[(have_family.PassengerId.isin([193,722])) ,["Group_size","Ave_Survival"]] = [2,0.5]
VER = 1
SEED = 42
seed_everything(SEED)
 xgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1913
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/m5-simple-fe/grid_part_1.pkl'
PRICE = '.. /input/m5-simple-fe/grid_part_2.pkl'
CALENDAR = '.. /input/m5-simple-fe/grid_part_3.pkl'
LAGS = '.. /input/m5-lags-features/lags_df_28.pkl'
MEAN_ENC = '.. /input/m5-custom-features/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/m5-xgb-aux-models/results/'
STORES_IDS = pd.read_csv(ORIGINAL+'sales_train_validation.csv')['store_id']
STORES_IDS = list(STORES_IDS.unique())
 cat_cols = ['item_id', 'dept_id', 'cat_id', 'event_name_1', 'event_type_1', 'event_name_2',
'event_type_2', 'snap_CA', 'snap_TX', 'snap_WI']
SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )<define_variables>
have_family.loc[have_family.PassengerId==443,["Group_size",'Ave_Survival']] = [2,0]
have_family.loc[have_family.PassengerId==1106,["Group_size",'Ave_Survival']] = [7,0]
<drop_column>
get_from_id_tic2(have_family,need_more_check,maybe_wrong )
grid_df, features_columns = get_data_by_store('CA_1')
 del grid_df
MODEL_FEATURES = features_columns<categorify>
have_family.loc[have_family.PassengerId.isin([41,893,1296]),["SibSp","Family_size"]] = [0,1]
combine.loc[combine.PassengerId.isin([41,893,1296]),["SibSp","Family_size"]] = [0,1]
have_family.loc[have_family.PassengerId.isin([69,1106]),["Group_size",'Ave_Survival']] = [7,0.125]
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle(AUX_MODELS + 'test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test<create_dataframe>
have_family = have_family[have_family.Family_size!=1]
passid_list = list(have_family.PassengerId.values)
 group_size_list = list(have_family.Group_size.values)
 ave_survival = list(have_family.Ave_Survival.values)
 combine.loc[combine.PassengerId.isin(passid_list),"Group_size"] = group_size_list
combine.loc[combine.PassengerId.isin(passid_list),"Ave_Survival"] = ave_survival
all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 for store_id in STORES_IDS:
model_path = 'xgb_model_'+store_id+'_v'+str(VER)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb'))
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 temp_df = grid_df[mask][MODEL_FEATURES].copy()
for c in cat_cols:
temp_df[c] = temp_df[c].cat.codes
test_data = xgb.DMatrix(temp_df)
 del temp_df
test_data.save_binary('test_data.bin')
 test_data = xgb.DMatrix('test_data.bin')
 base_test[TARGET][mask] = estimator.predict(test_data)
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
del temp_df
all_preds = all_preds.reset_index(drop=True)
 all_preds<save_to_csv>
all_passenger = []
for i,j in combine[(combine.Surname_Counts>=2)].groupby(["Ticket2","Surname"]):
if len(j)>=2:
j_passid = list(j.PassengerId.values)
 all_passenger = all_passenger+j_passid
same_surname = list(combine[(combine.PassengerId.isin(all_passenger)) &(combine.Group_size.isna())]["PassengerId"].values)
 print(len(same_surname))
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('submission_v'+str(VER)+'.csv', index=False )<set_options>
all_passenger = []
for i,j in combine[(combine.Maiden_Counts>=2)].groupby(["Ticket2","Maiden"]):
if len(j)>=2:
j_passid = list(j.PassengerId.values)
 all_passenger = all_passenger+j_passid
same_maiden = list(combine[(combine.PassengerId.isin(all_passenger)) &(combine.Group_size.isna())]["PassengerId"].values)
 print(same_maiden )
%pylab inline 
<import_modules>
combine[combine.Ticket=="31352"][['Age', 'Embarked', 'Parch', 'SibSp','Ticket','Surname', 'Maiden','Family_size','Pclass_Gender',
'Cabin_Title',"Surname_Counts","Group_size","Survived"]]
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelEncoder<import_modules>
team = combine[(combine.Ticket=="31352")|(( combine.Surname=="Ware")&(combine.Family_size!=1)) ]
combine.loc[(combine.PassengerId.isin([1170,1254])) ,"Group_size"] = 2
combine.loc[(combine.PassengerId.isin([1170,1254])) ,"Ave_Survival"] = team.Survived.mean()
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation
from keras.utils.np_utils import to_categorical
from keras.callbacks import EarlyStopping<load_from_csv>
same_surname.remove(1254)
 full_gs_and_as(dataset = combine,id_list=same_surname,ticket_kind = "Ticket2",check_list = [],with_friend=[],count_maiden = False )
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )<categorify>
combine[(combine.PassengerId.isin([115, 245, 303, 598, 693, 827])) ][["Pclass_Gender","Survived","Ticket"]]
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape )<normalization>
combine.loc[(combine.Ticket.isin(["2627","2694"])) ,"Group_size"] = combine[(combine.Ticket.isin(["2627","2694"])) ].shape[0]
combine.loc[(combine.Ticket.isin(["2627","2694"])) ,"Ave_Survival"] = combine[(combine.Ticket.isin(["2627","2694"])) ]["Survived"].mean()
combine.loc[(combine.Ticket.isin(["LINE"])) ,"Group_size"] = combine[(combine.Ticket.isin(["LINE"])) ].shape[0]
combine.loc[(combine.Ticket.isin(["LINE"])) ,"Ave_Survival"] = combine[(combine.Ticket.isin(["LINE"])) ]["Survived"].mean()
combine.loc[(combine.Ticket.isin(["1601"])) ,"Group_size"] = combine[(combine.Ticket.isin(["1601"])) ].shape[0]
combine.loc[(combine.Ticket.isin(["1601"])) ,"Ave_Survival"] = combine[(combine.Ticket.isin(["1601"])) ]["Survived"].mean()
X = StandardScaler().fit(data ).transform(data)
 print(X.shape )<categorify>
surname_list = list(combine.Surname.values)
 common_name = []
for maiden in combine.Maiden.values:
if maiden in surname_list:
common_name = common_name+[maiden]
print(len(common_name))
print(common_name[:5] )
y_cat = to_categorical(y)
 print(y_cat.shape )<choose_model_class>
possible_cousin = combine[(( combine.Surname.isin(common_name)) |(combine.Maiden.isin(common_name)))
 &(combine.Family_size==1)&(combine.Group_size.isna())]
possible_cousin_id = []
for i,j in possible_cousin.groupby(["Ticket2"]):
if len(j)>=2:
j_passid = list(j.PassengerId.values)
 possible_cousin_id = possible_cousin_id+[j_passid]
print(possible_cousin_id )
2
model = Sequential()
model.add(Dense(600,input_dim=192, init='uniform', activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(300, activation='sigmoid'))
model.add(Dropout(0.3))
model.add(Dense(99, activation='softmax'))<choose_model_class>
combine[(combine.PassengerId.isin([41, 104, 714, 877])) ][["Surname","Maiden"]]
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"] )<train_model>
combine[(combine.PassengerId.isin([41, 714])) ][['Age',"Cabin_Title", 'Embarked','Pclass_Gender',"Surname",'Maiden','Ticket',"Survived"]]
early_stopping = EarlyStopping(monitor='val_loss', patience=280)
 history = model.fit(X,y_cat,batch_size=192,
nb_epoch=800 ,verbose=0, validation_split=0.1, callbacks=[early_stopping] )<train_model>
combine.loc[combine.PassengerId.isin([41, 714]),["Group_size",'Ave_Survival']] = [2,0]
print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print()
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))<load_from_csv>
check_list = [[146, 550, 1068, 1222], [270, 326, 374, 1206], [280, 747, 1065, 1284], [291, 742, 988], [298, 306, 499, 709, 1033, 1198],
[308, 506, 1306], [381, 558, 701, 717, 1094], [538, 545, 1131]]
fillfull_p2(combine,check_list,check_more = [] )
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = StandardScaler().fit(test ).transform(test)
 yPred = model.predict_proba(test )<create_dataframe>
survival = combine[(combine.Pclass_Gender=="P3-Man")&(combine.Ave_Survival==0.5)]["Survived"].mean()
shape = combine[(combine.Pclass_Gender=="P3-Man")&(combine.Ave_Survival==0.5)&(combine.Survived.notna())].shape[0]
print(survival)
 print(shape )
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))<save_to_csv>
survival = combine[(combine.Pclass_Gender=="P1-Man")&(combine.Ave_Survival==0.5)]["Survived"].mean()
shape = combine[(combine.Pclass_Gender=="P1-Man")&(combine.Ave_Survival==0.5)&(combine.Survived.notna())].shape[0]
print(survival)
 print(shape )
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )<set_options>
survival =combine[(combine.Pclass_Gender=="P3-Boy")&(combine.Ave_Survival==0.5)]["Survived"].mean()
shape = combine[(combine.Pclass_Gender=="P3-Boy")&(combine.Ave_Survival==0.5)&(combine.Survived.notna())].shape[0]
print(survival)
 print(shape )
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline 
<load_pretrained>
combine[(combine.Pclass_Gender=="P3-Boy")&(combine.Ave_Survival==0.5)
 &(combine.Survived.isna())][["PassengerId","Ticket","Ticket_Counts"]]
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )<load_from_csv>
combine[combine.Ticket=="2673"][["PassengerId","Age","Pclass_Gender","Survived","Ave_Survival"]]
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()<feature_engineering>
combine.loc[(combine.Pclass_Gender.isin(["P3-Woman"])) &(combine.Ave_Survival>=0.5),"If_Survived"] = 1
combine.loc[(combine.Pclass_Gender.isin(["P3-Woman"])) &(combine.Ave_Survival<0.5),"If_Survived"] = 0
combine.loc[(combine.Pclass_Gender.isin(["P1-Man","P3-Man"])&(combine.Ave_Survival>0.5)) ,"If_Survived"] = 1
combine.loc[(combine.Pclass_Gender.isin(["P1-Man","P3-Man"])&(combine.Ave_Survival<=0.5)) ,"If_Survived"] = 0
combine.loc[(( combine.Pclass_Gender.isin(["P3-Boy"])) &(combine.Ave_Survival>0.5)) ,"If_Survived"] = 1
combine.loc[(( combine.Pclass_Gender.isin(["P3-Boy"])) &(combine.Ave_Survival<0.5)) ,"If_Survived"] = 0
combine.loc[(combine.PassengerId.isin([1053,1199,1309])) ,"If_Survived"] = 1
combine.loc[(combine.PassengerId.isin([1046,1084,1231,1236,1271,1284])) ,"If_Survived"] = 0
combine.loc[(combine.Group_size.isna()),"Group_size"] = 1
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True )<compute_test_metric>
combine[(combine.Pclass_Gender.isin(["P1-Man","P3-Man"]))
&(combine.Survived.isna())&(combine.If_Survived==1)][["Pclass_Gender","Group_size","Ave_Survival","Ticket"]]
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )<load_pretrained>
combine.loc[(combine.If_Survived==0),"Predict"] = 0
combine.loc[(combine.If_Survived==1),"Predict"] = 1
combine.loc[(combine.PassengerId.isin([1036,1047,1210])) ,"Predict"] = 0
learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png')) 
<load_pretrained>
combine = combine[["PassengerId","Age","Embarked","Survived","Pclass_Gender","Fare_PP","Cabin_Title","Predict"]]
learn.load('abcdef');<compute_train_metric>
remain_train = combine[(combine.Predict.isna())&(combine.Survived.notna())]
remain_test = combine[(combine.Predict.isna())&(combine.Survived.isna())]
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<save_to_csv>
print("Pclass_Gender distribution in train data:",remain_train.Pclass_Gender.value_counts())
 print("-"*50)
 print("Pclass_Gender distribution in test data:",remain_test.Pclass_Gender.value_counts() )
def run_subm(learn=learn, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )<install_modules>
survival_in_remain = remain_train[(remain_train.Pclass_Gender=="P1-Man")]["Survived"].mean()
survival_in_all = combine[(combine.Pclass_Gender=="P1-Man")]["Survived"].mean()
print(round(survival_in_all,2),round(survival_in_remain,2))
!pip install -U '.. /input/install/efficientnet-0.0.3-py2.py3-none-any.whl'<load_from_csv>
survival_in_remain = remain_train[(remain_train.Pclass_Gender=="P3-Man")]["Survived"].mean()
survival_in_all = combine[(combine.Pclass_Gender=="P3-Man")]["Survived"].mean()
print(round(survival_in_all,2),round(survival_in_remain,2))
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv' )<split>
combine.loc[(combine.Predict.isna())&(combine.Survived.isna())&(combine.Pclass_Gender=="P3-Woman"),"Predict"] = 1
combine.loc[(combine.Predict.isna())&(combine.Survived.isna()),"Predict"]=0
y = to_categorical(y, num_classes=NUM_CLASSES)
 train_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,
stratify=y, random_state=8)
 print(train_x.shape)
 print(train_y.shape)
 print(valid_x.shape)
 print(valid_y.shape )<define_variables>
final_submit = pd.DataFrame({"PassengerId":combine[combine.Survived.isna() ].PassengerId.values,
"Survived":combine[combine.Survived.isna() ].Predict.values})
 final_submit["Survived"] = final_submit.Survived.astype("int64")
 final_submit.tail(2 )
<load_pretrained><EOS> 
final_submit.to_csv("Titanic.csv",index = False )