Kaggle competition: Titanic - Machine Learning from Disaster
class BertQAModel(TFBertPreTrainedModel):
DROPOUT_RATE = 0.1
NUM_HIDDEN_STATES = 2
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.bert = TFBertMainLayer(config, name="bert")
 self.concat = L.Concatenate()
self.dropout = L.Dropout(self.DROPOUT_RATE)
 self.qa_outputs = L.Dense(
 config.num_labels,
kernel_initializer=TruncatedNormal(stddev=config.initializer_range),
dtype='float32',
name="qa_outputs")
 @tf.function
def call(self, inputs, **kwargs):
_, _, hidden_states = self.bert(inputs, **kwargs)
 hidden_states = self.concat([
hidden_states[-i] for i in range(1, self.NUM_HIDDEN_STATES+1)
 ])
 hidden_states = self.dropout(hidden_states, training=kwargs.get("training", False))
logits = self.qa_outputs(hidden_states)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, axis=-1)
 end_logits = tf.squeeze(end_logits, axis=-1)
 return start_logits, end_logits
def train(model, dataset, loss_fn, optimizer):
@tf.function
def train_step(model, inputs, y_true, loss_fn, optimizer):
with tf.GradientTape() as tape:
y_pred = model(inputs, training=True)
 loss = loss_fn(y_true[0], y_pred[0])
 loss += loss_fn(y_true[1], y_pred[1])
 scaled_loss = optimizer.get_scaled_loss(loss)
 scaled_gradients = tape.gradient(scaled_loss, model.trainable_variables)
 gradients = optimizer.get_unscaled_gradients(scaled_gradients)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss, y_pred
epoch_loss = 0.
 for batch_num, sample in enumerate(dataset):
loss, y_pred = train_step(
 model, sample[:3], sample[4:6], loss_fn, optimizer)
 epoch_loss += loss
print(
 f"training...batch {batch_num+1:03d} : "
f"train loss {epoch_loss/(batch_num+1):.3f} ",
end='\r')
 def predict(model, dataset, loss_fn, optimizer):
@tf.function
def predict_step(model, inputs):
return model(inputs)
 def to_numpy(*args):
out = []
for arg in args:
if arg.dtype == tf.string:
arg = [s.decode('utf-8')for s in arg.numpy() ]
out.append(arg)
 else:
arg = arg.numpy()
out.append(arg)
 return out
offset = tf.zeros([0, MAX_SEQUENCE_LENGTH, 2], dtype=tf.dtypes.int32)
 text = tf.zeros([0,], dtype=tf.dtypes.string)
 selected_text = tf.zeros([0,], dtype=tf.dtypes.string)
 sentiment = tf.zeros([0,], dtype=tf.dtypes.string)
 pred_start = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 pred_end = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 for batch_num, sample in enumerate(dataset):
print(f"predicting...batch {batch_num+1:03d}"+" "*20, end='\r')
 y_pred = predict_step(model, sample[:3])
 pred_start = tf.concat(( pred_start, y_pred[0]), axis=0)
 pred_end = tf.concat(( pred_end, y_pred[1]), axis=0)
 offset = tf.concat(( offset, sample[3]), axis=0)
 text = tf.concat(( text, sample[6]), axis=0)
 selected_text = tf.concat(( selected_text, sample[7]), axis=0)
 sentiment = tf.concat(( sentiment, sample[8]), axis=0)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
to_numpy(pred_start, pred_end, text, selected_text, sentiment, offset)
 return pred_start, pred_end, text, selected_text, sentiment, offset
def decode_prediction(pred_start, pred_end, text, offset, sentiment):
def decode(pred_start, pred_end, text, offset):
decoded_text = ""
for i in range(pred_start, pred_end+1):
decoded_text += text[offset[i][0]:offset[i][1]]
if(i+1)< len(offset)and offset[i][1] < offset[i+1][0]:
decoded_text += " "
return decoded_text
decoded_predictions = []
for i in range(len(text)) :
if sentiment[i] == "neutral" or len(text[i].split())< 2:
decoded_text = text[i]
else:
idx_start = np.argmax(pred_start[i])
 idx_end = np.argmax(pred_end[i])
 if idx_start > idx_end:
idx_end = idx_start
decoded_text = str(decode(idx_start, idx_end, text[i], offset[i]))
if len(decoded_text)== 0:
decoded_text = text[i]
decoded_predictions.append(decoded_text)
 return decoded_predictions
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c)) 
<choose_model_class>
df_train = pd.read_csv('.. /input/train.csv', index_col='PassengerId')
 df_test = pd.read_csv('.. /input/test.csv', index_col='PassengerId')
 df_gender_sub = pd.read_csv(".. /input/gender_submission.csv", index_col='PassengerId' )
num_folds = 4
num_epochs = 3
batch_size = 32
learning_rate = 3e-5
optimizer = tf.keras.optimizers.Adam(learning_rate)
 optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, 'dynamic')
 config = BertConfig(output_hidden_states=True, num_labels=2)
 BertQAModel.DROPOUT_RATE = 0.2
BertQAModel.NUM_HIDDEN_STATES = 2
model = BertQAModel.from_pretrained(PATH, config=config)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 kfold = model_selection.StratifiedKFold(
 n_splits=num_folds, shuffle=True, random_state=42)
 test_preds_start = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 test_preds_end = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 for fold_num,(train_idx, valid_idx)in enumerate(
 kfold.split(X=train_df.text, y=train_df.sentiment.values)) :
print("
fold %02d" %(fold_num+1))
train_dataset = TweetSentimentDataset.create(
 train_df.iloc[train_idx], batch_size, shuffle_buffer_size=2048)
 valid_dataset = TweetSentimentDataset.create(
 train_df.iloc[valid_idx], batch_size, shuffle_buffer_size=-1)
 test_dataset = TweetSentimentDataset.create(
 test_df, batch_size, shuffle_buffer_size=-1)
 best_score = float('-inf')
 for epoch_num in range(num_epochs):
print("
epoch %03d" %(epoch_num+1))
train(model, train_dataset, loss_fn, optimizer)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
predict(model, valid_dataset, loss_fn, optimizer)
 selected_text_pred = decode_prediction(
 pred_start, pred_end, text, offset, sentiment)
 jaccards = []
for i in range(len(selected_text)) :
jaccards.append(
 jaccard(selected_text[i], selected_text_pred[i]))
score = np.mean(jaccards)
 print(f"valid jaccard epoch {epoch_num+1:03d}: {score}"+" "*15)
 if score > best_score:
best_score = score
test_pred_start, test_pred_end, test_text, _, test_sentiment, test_offset = \
predict(model, test_dataset, loss_fn, optimizer)
 test_preds_start += test_pred_start
test_preds_end += test_pred_end
session = tf.compat.v1.get_default_session()
graph = tf.compat.v1.get_default_graph()
del session, graph, model
model = BertQAModel.from_pretrained(PATH, config=config)
 selected_text_pred = decode_prediction(
 test_preds_start, test_preds_end, test_text, test_offset, test_sentiment)
 submission_df.loc[:, 'selected_text'] = selected_text_pred
submission_df.to_csv("submission.csv", index=False )<set_options>
Survived = df_train.loc[:,'Survived']
df_train = df_train.drop(['Survived'], axis=1 ).copy()
train_index = df_train.index
test_index = df_test.index
df_all = pd.concat([df_train, df_test])
tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 tf.config.optimizer.set_jit(True)
 tf.config.optimizer.set_experimental_options(
 {"auto_mixed_precision": True} )<load_from_csv>
def nullAnalysis(df):
tab_info=pd.DataFrame(df.dtypes ).T.rename(index={0:'column type'})
 tab_info=tab_info.append(pd.DataFrame(df.isnull().sum() ).T.rename(index={0:'null values(nb)'}))
tab_info=tab_info.append(pd.DataFrame(df.isnull().sum() /df.shape[0]*100)
.T.rename(index={0:'null values(%)'}))
return tab_info
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df.dropna(inplace=True)
 test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df.loc[:, "selected_text"] = test_df.text.values
submission_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 print("train shape =", train_df.shape)
 print("test shape =", test_df.shape)
 PATH = ".. /input/bert-base-uncased/"
MAX_SEQUENCE_LENGTH = 128
TOKENIZER = BertWordPieceTokenizer(f"{PATH}/vocab.txt", lowercase=True, add_special_tokens=False)
 train_df.head(10 )<categorify>
nullAnalysis(df_all )
def preprocess(tweet, selected_text, sentiment):

 tweet = tweet.decode('utf-8')
 selected_text = selected_text.decode('utf-8')
 sentiment = sentiment.decode('utf-8')
 tweet = " ".join(str(tweet ).split())
 selected_text = " ".join(str(selected_text ).split())
 idx_start, idx_end = None, None
for index in(i for i, c in enumerate(tweet)if c == selected_text[0]):
if tweet[index:index+len(selected_text)] == selected_text:
idx_start = index
idx_end = index + len(selected_text)
 break
intersection = [0] * len(tweet)
 if idx_start != None and idx_end != None:
for char_idx in range(idx_start, idx_end):
intersection[char_idx] = 1
enc = TOKENIZER.encode(tweet)
 input_ids_orig, offsets = enc.ids, enc.offsets
target_idx = []
for i,(o1, o2)in enumerate(offsets):
if sum(intersection[o1: o2])> 0:
target_idx.append(i)
 target_start = target_idx[0]
target_end = target_idx[-1]
sentiment_map = {
'positive': 3893,
'negative': 4997,
'neutral': 8699,
}
input_ids = [101] + [sentiment_map[sentiment]] + [102] + input_ids_orig + [102]
input_type_ids = [0, 0, 0] + [1] *(len(input_ids_orig)+ 1)
 attention_mask = [1] *(len(input_ids_orig)+ 4)
 offsets = [(0, 0),(0, 0),(0, 0)] + offsets + [(0, 0)]
target_start += 3
target_end += 3
padding_length = MAX_SEQUENCE_LENGTH - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 input_type_ids = input_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 elif padding_length < 0:
input_ids = input_ids[:padding_length-1] + [102]
attention_mask = attention_mask[:padding_length-1] + [1]
input_type_ids = input_type_ids[:padding_length-1] + [1]
offsets = offsets[:padding_length-1] + [(0, 0)]
if target_start >= MAX_SEQUENCE_LENGTH:
target_start = MAX_SEQUENCE_LENGTH - 1
if target_end >= MAX_SEQUENCE_LENGTH:
target_end = MAX_SEQUENCE_LENGTH - 1
return(
 input_ids, attention_mask, input_type_ids, offsets,
target_start, target_end, tweet, selected_text, sentiment, 
)
 class TweetSentimentDataset(tf.data.Dataset):
OUTPUT_TYPES =(
 tf.dtypes.int32, tf.dtypes.int32, tf.dtypes.int32,
tf.dtypes.int32, tf.dtypes.float32, tf.dtypes.float32,
tf.dtypes.string, tf.dtypes.string, tf.dtypes.string, 
)
 OUTPUT_SHAPES =(
(MAX_SEQUENCE_LENGTH,),(MAX_SEQUENCE_LENGTH,),(MAX_SEQUENCE_LENGTH,), 
(MAX_SEQUENCE_LENGTH, 2),() ,() , 
() ,() ,() , 
)
 def _generator(tweet, selected_text, sentiment):
for tw, st, se in zip(tweet, selected_text, sentiment):
yield preprocess(tw, st, se)
 def __new__(cls, tweet, selected_text, sentiment):
return tf.data.Dataset.from_generator(
 cls._generator,
output_types=cls.OUTPUT_TYPES,
output_shapes=cls.OUTPUT_SHAPES,
args=(tweet, selected_text, sentiment)
)
 @staticmethod
def create(dataframe, batch_size, shuffle_buffer_size=-1):
dataset = TweetSentimentDataset(
 dataframe.text.values,
dataframe.selected_text.values,
dataframe.sentiment.values 
)
 dataset = dataset.cache()
if shuffle_buffer_size != -1:
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset 
<choose_model_class>
df_all.groupby('Pclass')['Age'].agg('mean' )
class BertQAModel(TFBertPreTrainedModel):
DROPOUT_RATE = 0.1
NUM_HIDDEN_STATES = 2
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.bert = TFBertMainLayer(config, name="bert")
 self.concat = L.Concatenate()
self.dropout = L.Dropout(self.DROPOUT_RATE)
 self.qa_outputs = L.Dense(
 config.num_labels,
kernel_initializer=TruncatedNormal(stddev=config.initializer_range),
dtype='float32',
name="qa_outputs")
 @tf.function
def call(self, inputs, **kwargs):
_, _, hidden_states = self.bert(inputs, **kwargs)
 hidden_states = self.concat([
hidden_states[-i] for i in range(1, self.NUM_HIDDEN_STATES+1)
 ])
 hidden_states = self.dropout(hidden_states, training=kwargs.get("training", False))
logits = self.qa_outputs(hidden_states)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, axis=-1)
 end_logits = tf.squeeze(end_logits, axis=-1)
 return start_logits, end_logits
def train(model, dataset, loss_fn, optimizer):
@tf.function
def train_step(model, inputs, y_true, loss_fn, optimizer):
with tf.GradientTape() as tape:
y_pred = model(inputs, training=True)
 loss = loss_fn(y_true[0], y_pred[0])
 loss += loss_fn(y_true[1], y_pred[1])
 scaled_loss = optimizer.get_scaled_loss(loss)
 scaled_gradients = tape.gradient(scaled_loss, model.trainable_variables)
 gradients = optimizer.get_unscaled_gradients(scaled_gradients)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss, y_pred
epoch_loss = 0.
 for batch_num, sample in enumerate(dataset):
loss, y_pred = train_step(
 model, sample[:3], sample[4:6], loss_fn, optimizer)
 epoch_loss += loss
print(
 f"training...batch {batch_num+1:03d} : "
f"train loss {epoch_loss/(batch_num+1):.3f} ",
end='\r')
 def predict(model, dataset, loss_fn, optimizer):
@tf.function
def predict_step(model, inputs):
return model(inputs)
 def to_numpy(*args):
out = []
for arg in args:
if arg.dtype == tf.string:
arg = [s.decode('utf-8')for s in arg.numpy() ]
out.append(arg)
 else:
arg = arg.numpy()
out.append(arg)
 return out
offset = tf.zeros([0, MAX_SEQUENCE_LENGTH, 2], dtype=tf.dtypes.int32)
 text = tf.zeros([0,], dtype=tf.dtypes.string)
 selected_text = tf.zeros([0,], dtype=tf.dtypes.string)
 sentiment = tf.zeros([0,], dtype=tf.dtypes.string)
 pred_start = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 pred_end = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 for batch_num, sample in enumerate(dataset):
print(f"predicting...batch {batch_num+1:03d}"+" "*20, end='\r')
 y_pred = predict_step(model, sample[:3])
 pred_start = tf.concat(( pred_start, y_pred[0]), axis=0)
 pred_end = tf.concat(( pred_end, y_pred[1]), axis=0)
 offset = tf.concat(( offset, sample[3]), axis=0)
 text = tf.concat(( text, sample[6]), axis=0)
 selected_text = tf.concat(( selected_text, sample[7]), axis=0)
 sentiment = tf.concat(( sentiment, sample[8]), axis=0)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
to_numpy(pred_start, pred_end, text, selected_text, sentiment, offset)
 return pred_start, pred_end, text, selected_text, sentiment, offset
def decode_prediction(pred_start, pred_end, text, offset, sentiment):
def decode(pred_start, pred_end, text, offset):
decoded_text = ""
for i in range(pred_start, pred_end+1):
decoded_text += text[offset[i][0]:offset[i][1]]
if(i+1)< len(offset)and offset[i][1] < offset[i+1][0]:
decoded_text += " "
return decoded_text
decoded_predictions = []
for i in range(len(text)) :
if sentiment[i] == "neutral" or len(text[i].split())< 2:
decoded_text = text[i]
else:
idx_start = np.argmax(pred_start[i])
 idx_end = np.argmax(pred_end[i])
 if idx_start > idx_end:
idx_end = idx_start
decoded_text = str(decode(idx_start, idx_end, text[i], offset[i]))
if len(decoded_text)== 0:
decoded_text = text[i]
decoded_predictions.append(decoded_text)
 return decoded_predictions
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c)) 
<choose_model_class>
df_all.loc[(df_all['Age'].isnull())&(df_all['Pclass'] == 1), ['Age']] = round(df_all.groupby('Pclass')['Age'].agg('mean')[1],0)
 df_all.loc[(df_all['Age'].isnull())&(df_all['Pclass'] == 2), ['Age']] = round(df_all.groupby('Pclass')['Age'].agg('mean')[2],0)
 df_all.loc[(df_all['Age'].isnull())&(df_all['Pclass'] == 3), ['Age']] = round(df_all.groupby('Pclass')['Age'].agg('mean')[3],0 )
num_folds = 4
num_epochs = 3
batch_size = 32
learning_rate = 3e-5
optimizer = tf.keras.optimizers.Adam(learning_rate)
 optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, 'dynamic')
 config = BertConfig(output_hidden_states=True, num_labels=2)
 BertQAModel.DROPOUT_RATE = 0.2
BertQAModel.NUM_HIDDEN_STATES = 2
model = BertQAModel.from_pretrained(PATH, config=config)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 kfold = model_selection.StratifiedKFold(
 n_splits=num_folds, shuffle=True, random_state=42)
 test_preds_start = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 test_preds_end = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 for fold_num,(train_idx, valid_idx)in enumerate(
 kfold.split(X=train_df.text, y=train_df.sentiment.values)) :
print("
fold %02d" %(fold_num+1))
train_dataset = TweetSentimentDataset.create(
 train_df.iloc[train_idx], batch_size, shuffle_buffer_size=2048)
 valid_dataset = TweetSentimentDataset.create(
 train_df.iloc[valid_idx], batch_size, shuffle_buffer_size=-1)
 test_dataset = TweetSentimentDataset.create(
 test_df, batch_size, shuffle_buffer_size=-1)
 best_score = float('-inf')
 for epoch_num in range(num_epochs):
print("
epoch %03d" %(epoch_num+1))
train(model, train_dataset, loss_fn, optimizer)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
predict(model, valid_dataset, loss_fn, optimizer)
 selected_text_pred = decode_prediction(
 pred_start, pred_end, text, offset, sentiment)
 jaccards = []
for i in range(len(selected_text)) :
jaccards.append(
 jaccard(selected_text[i], selected_text_pred[i]))
score = np.mean(jaccards)
 print(f"valid jaccard epoch {epoch_num+1:03d}: {score}"+" "*15)
 if score > best_score:
best_score = score
test_pred_start, test_pred_end, test_text, _, test_sentiment, test_offset = \
predict(model, test_dataset, loss_fn, optimizer)
 test_preds_start += test_pred_start
test_preds_end += test_pred_end
session = tf.compat.v1.get_default_session()
graph = tf.compat.v1.get_default_graph()
del session, graph, model
model = BertQAModel.from_pretrained(PATH, config=config)
 selected_text_pred = decode_prediction(
 test_preds_start, test_preds_end, test_text, test_offset, test_sentiment)
 submission_df.loc[:, 'selected_text'] = selected_text_pred
submission_df.to_csv("submission.csv", index=False )<import_modules>
df_all[df_all['Fare'].isnull() ]
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from scipy.sparse import hstack
from scipy.special import logit, expit<load_from_csv>
df_all.groupby('Pclass', as_index=False)['Fare'].agg('mean' )
train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ' )<define_variables>
df_all.loc[1044,['Fare']] = 13.30
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']<concatenate>
print("Count of cabins with nan data: ")
 df_all.loc[(df_all.loc[:,'Cabin'].isnull())== True]['Name'].count()
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )<feature_engineering>
df_all.groupby('Pclass' ).agg('count')[['Name','Cabin']]
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 1),
max_features=10000)
 word_vectorizer.fit(all_text)
 train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text )<feature_engineering>
( df_all.groupby('Pclass' ).agg('count')['Cabin'] / df_all.groupby('Pclass' ).agg('count')['Name'])*100
char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 5),
max_features=30000)
 char_vectorizer.fit(all_text)
 train_char_features = char_vectorizer.transform(train_text)
 test_char_features = char_vectorizer.transform(test_text )<concatenate>
df_all[df_all['Cabin'].str.contains(' ', regex=False)== True].sort_values(by='Cabin' )
train_features = hstack([train_char_features, train_word_features])
 test_features = hstack([test_char_features, test_word_features])
<find_best_model_class>
df_cabin_expand = df_all.loc[:,'Cabin'].str.split(' ', expand=True)
 df_cabin_expand[df_cabin_expand.loc[:,1].isnull() == False].groupby([0] ).count()
losses = []
predictions = {'id': test['id']}
for class_name in class_names:
train_target = train[class_name]
classifier = LogisticRegression(solver='sag')
 cv_loss = np.mean(cross_val_score(classifier, train_features, train_target, cv=3, scoring='roc_auc'))
losses.append(cv_loss)
 print('CV score for class {} is {}'.format(class_name, cv_loss))
classifier.fit(train_features, train_target)
 predictions[class_name] = classifier.predict_proba(test_features)[:, 1]
print('Total CV score is {}'.format(np.mean(losses)) )<save_to_csv>
df_all[df_all['Embarked'].isnull() ]
submission = pd.DataFrame.from_dict(predictions)
 submission.to_csv('word_submission.csv', index=False )<import_modules>
df_all['Embarked'] = df_all['Embarked'].fillna(method='bfill' )
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import BernoulliNB
from sklearn.linear_model import LogisticRegression
from sklearn import linear_model
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import cross_val_score
from scipy.sparse import hstack 
<import_modules>
df_name_salutation = df_all.loc[:,'Name'].str.split(' ', expand=True ).copy()
df_name_salutation.groupby(1 ).count()
stopwords = nltk.corpus.stopwords.words('english' )<set_options>
df_newsal_1 = df_name_salutation[df_name_salutation[1].str.contains('.', regex=False)][1]
df_newsal_2 = df_name_salutation[df_name_salutation[2].str.contains('.', regex=False)][2]
df_newsal_3 = df_name_salutation[(df_name_salutation[3].isnull() == False)&(df_name_salutation[3].str.contains('.', regex=False)) ][3]
df_newsal_2 = df_newsal_2.rename(1)
 df_newsal_3 = df_newsal_3.rename(1)
 df_newsal = df_newsal_1.append([df_newsal_2, df_newsal_3])
 df_newsal = df_newsal.rename('Salutation')
 df_all = pd.concat([df_all,df_newsal],axis=1 )
sns.set_style("dark" )<load_from_csv>
df_all.groupby('Salutation' ).count()
train = pd.read_csv('.. /input/train.csv', error_bad_lines=False ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv', error_bad_lines=False ).fillna(' ')
 subm = pd.read_csv('.. /input/sample_submission.csv' )<data_type_conversions>
df_all.loc[:,'Family'] =(( df_all['SibSp'] > 0)|(df_all['Parch'] > 0)).replace(True, 1, inplace=False)
 df_all.loc[:,'Family'] = df_all.loc[:,'Family'].astype(int )
vectorizer = TfidfVectorizer(ngram_range=(1, 2),
max_df=0.5,
min_df=4,
max_features=1000)
 vector_space_model = vectorizer.fit_transform(train['comment_text'].values.astype('U' ).tolist())
 n_comments = vector_space_model.shape[0]
print('%d Total Comments' % n_comments )<train_model>
df_familynames = df_all.loc[:,'Name'].str.split(' ', expand=True ).copy()
l_singleLastname = df_familynames[(df_familynames[0].str.contains(',', regex=False)==True)].index
l_doubleLastname = df_familynames[(df_familynames[0].str.contains(',', regex=False)==False)
 &(df_familynames[1].str.contains(',', regex=False)==True)].index
l_doubleLastnameSpec = df_familynames[(df_familynames[0].str.contains(',', regex=False)==False)
 &(df_familynames[1].str.contains(',', regex=False)==False)
 &(df_familynames[2].str.contains(',', regex=False)==True)].index
df_singleLastname = df_familynames.loc[l_singleLastname,0]
df_doubleLastname =(df_familynames.loc[l_doubleLastname,0]
+ ' ' + df_familynames.loc[l_doubleLastname,1])
 df_doubleLastnameSpec =(df_familynames.loc[l_doubleLastnameSpec,0]
+ ' ' + df_familynames.loc[l_doubleLastnameSpec,1]
+ ' ' + df_familynames.loc[l_doubleLastnameSpec,2])
 df_singleLastname = df_singleLastname.rename('Lastname')
 df_doubleLastname = df_doubleLastname.rename('Lastname')
 df_doubleLastnameSpec = df_doubleLastnameSpec.rename('Lastname')
 df_familynames['Lastname'] = df_singleLastname.append([df_doubleLastname, df_doubleLastnameSpec])
training_set_size = int(n_comments * 0.33)
 X = vector_space_model[:training_set_size,:]
Z = vector_space_model[training_set_size:vector_space_model.shape[0]-1,:]
print('%d comments for the estimation of the parameters and %d for the evaluation' % 
(X.shape[0], Z.shape[0]))<train_model>
df_lastname_count = df_familynames.groupby('Lastname', as_index=False ).count()
df_lastname_count = df_lastname_count.drop([1,2,3,4,5,6,7,8,9,10,11,12,13], axis=1)
 df_familynames = df_familynames.drop([0,1,2,3,4,5,6,7,8,9,10,11,12,13], axis=1)
 df_familynames = df_familynames.join(df_lastname_count.set_index('Lastname'), on='Lastname')
 df_familynames = df_familynames.rename(columns = {0: "Number_of_Familymembers"} )
X = X.toarray()
Y = train['toxic'][:training_set_size]
model = linear_model.BayesianRidge(verbose=True)
 model.fit(X, Y )<predict_on_test>
df_all = pd.merge(df_all , df_familynames, right_index=True, left_index=True)
 df_all['Lastname'] = df_all['Lastname'].str.rstrip(',' )
ground_truth = train['toxic'][training_set_size:vector_space_model.shape[0]-1]
prediction = model.predict(Z)
 prediction = binarize(prediction.reshape(-1, 1), 0.5 )<define_variables>
df_all.loc[df_all['Family'] == 0,'Number_of_Familymembers'] = 1
toxic_ids = [i for i, c in enumerate(prediction)if c == 1]
toxic_ids<predict_on_test>
df_all.groupby(['Parch'] ).agg('count' )
comment_id = toxic_ids[0]
print('Content of the comment:
%s
' % train['comment_text'][training_set_size+comment_id])
 print('Is this comment "toxic" according to the model?
%s' % str(model.predict(Z[comment_id,:])>0.5))<predict_on_test>
df_sal_distr = df_all.groupby('Salutation' ).count()
df_sal_distr.reset_index(level=0, inplace=True)
 df_sal_distr = df_sal_distr[['Salutation','Pclass']]
df_sal_distr = df_sal_distr.rename(columns = {'Pclass':"Salutation_Count"})
 df_sal_distr
comment_id = toxic_ids[1]
print('Content of the comment:
%s
' % train['comment_text'][training_set_size+comment_id])
 print('Is this comment "toxic" according to the model?
%s' % str(model.predict(Z[comment_id,:])>0.5))<predict_on_test>
df_survivalinfo = pd.concat([df_all.loc[train_index,:], Survived], axis=1 )
comment_id = toxic_ids[2]
print('Content of the comment:
%s
' % train['comment_text'][training_set_size+comment_id])
 print('Is this comment "toxic" according to the model?
%s' % str(model.predict(Z[comment_id,:])>0.5))<string_transform>
gp_survived_gender = df_survivalinfo.groupby(['Survived','Sex'])['Name'].count() [1]
gp_gender_survived = df_survivalinfo.groupby(['Sex','Survived'] ).count() ['Name']
gp_survived_yn = df_survivalinfo.groupby(['Survived'] ).agg('count')['Name']
gp_survival_total = df_survivalinfo.groupby(['Sex','Survived'] ).count().xs('Name', axis=1)
 sizes_suvinf = [gp_survived_yn[y] for y in range(len(gp_survived_yn)) ]
explode_suvinf =(0, 0.1)
 labels_sexinf = [gp_survived_gender.index[y] for y in range(len(gp_survived_gender.index)) ]
sizes_sexinf = [gp_survived_gender[y] for y in range(len(gp_survived_gender)) ]
explode_sexinf =(0, 0.1)
 labels_maleinf = [gp_gender_survived['male'].index[y] for y in range(len(gp_gender_survived['male'].index)) ]
sizes_maleinf = [gp_gender_survived['male'][y] for y in range(len(gp_gender_survived['male'])) ]
explode_maleinf =(0, 0.1)
 labels_femaleinf = [gp_gender_survived['female'].index[y] for y in range(len(gp_gender_survived['female'].index)) ]
sizes_femaleinf = [gp_gender_survived['female'][y] for y in range(len(gp_gender_survived['female'])) ]
explode_femaleinf =(0, 0.1)
 labels_totalsuvinf = [gp_survival_total.index[y] for y in range(len(gp_gender_survived.index)) ]
sizes_totalsuvinf = [gp_survival_total[y] for y in range(len(gp_survival_total)) ]
explode_totalsuvinf =(0.3, 0.0, 0.1, 0.0)
 font = {'color': 'black',
'weight': 'normal',
'fontsize':15
}
cmap = plt.get_cmap('tab20c')
 survivedcolor = cmap(np.array([5,1]))
survivorallcolor = cmap(np.array([9,10,0,1]))
gridsize =(4,2)
 fig1 = plt.figure(figsize=(14,10))
ax1 = plt.subplot2grid(gridsize,(0,0))
ax2 = plt.subplot2grid(gridsize,(0,1))
ax3 = plt.subplot2grid(gridsize,(1,0))
ax4 = plt.subplot2grid(gridsize,(1,1))
ax5 = plt.subplot2grid(gridsize,(2,0), colspan= 2, rowspan= 2)
 fig1.suptitle('Distributions of 891 Passengers(Trainingset)', fontsize=25)
 ax1.pie(sizes_suvinf,
explode=explode_suvinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivedcolor,
labeldistance=1.15,
pctdistance=0.55 
)
 ax1.axis('equal')
 ax1.legend(loc='upper left',fontsize='12',labels=('not survived', 'survived'))
ax1.set_title('Survivor Distribution', fontdict=font, fontsize=20)
 ax2.pie(sizes_sexinf,
explode=explode_sexinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=outer_colors,
labeldistance=1.15,
pctdistance=0.55)
 ax2.axis('equal')
 ax2.legend(loc='upper right', fontsize='12', labels=labels_sexinf)
 ax2.set_title('Gender Survivors', fontdict=font, fontsize=20)
 ax3.pie(sizes_maleinf,
explode=explode_maleinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivedcolor,
labeldistance=1.15,
pctdistance=0.55)
 ax3.legend(loc='lower left', fontsize='12', labels=['not survived','survived'])
 ax3.set_title('Male Survivors', fontdict=font, fontsize=20)
 ax3.axis('equal')
 ax4.pie(sizes_femaleinf,
explode=explode_femaleinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivedcolor,
labeldistance=1.15,
pctdistance=0.55)
 ax4.legend(loc='lower right', fontsize='12', labels=['not survived','survived'])
 ax4.set_title('Female Survivors', fontdict=font, fontsize=20)
 ax4.axis('equal')
 ax5.pie(sizes_totalsuvinf,
explode=explode_totalsuvinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivorallcolor,
labeldistance=1.15,
pctdistance=0.55)
 ax5.axis('equal')
 ax5.legend(loc='lower right', fontsize='12', labels=['female not survived','female survived','male not survived','male survived'])
 ax5.set_title('Overall Survivors', fontdict=font, fontsize=20)
 ax5.axis('equal')
 plt.show()
comment_text_list = train.apply(lambda row : nltk.word_tokenize(row['comment_text']),axis=1 )<feature_engineering>
ptbl = pd.DataFrame.pivot_table(df_survivalinfo, values=['Fare', 'Survived'], index=['Pclass'],
aggfunc={'Survived': ['sum'], 'Fare': [min,max,np.mean]})
 ptbl
rate_punctuation=0.7
rate_capital=0.7
def odd_comment(comment):
punctuation_count=0
capital_letter_count=0
total_letter_count=0
for token in comment:
if token in list(string.punctuation):
punctuation_count+=1
capital_letter_count+=sum(1 for c in token if c.isupper())
 total_letter_count+=len(token)
 return(( punctuation_count/len(comment)) >=rate_punctuation or 
(capital_letter_count/total_letter_count)>rate_capital)
 odd=comment_text_list.apply(odd_comment )<count_values>
df_survival_sal = df_survivalinfo.groupby(['Salutation','Survived'], as_index=False)['Name'].count()
df_survival_sal = df_survival_sal.rename(columns={"Name": "Total"})
 df_survival_sal_total = df_survival_sal.groupby('Salutation', as_index=False)['Total'].agg(sum)
 df_survival_sal_nsuv = pd.DataFrame(df_survival_sal[df_survival_sal['Survived'] == 0])
 df_survival_sal_nsuv = df_survival_sal_nsuv.rename(columns={'Total':'Total_notSurvived'})
 df_survival_sal_nsuv = df_survival_sal_nsuv[['Salutation','Total_notSurvived']]
df_survival_sal_suv = pd.DataFrame(df_survival_sal[df_survival_sal['Survived'] == 1])
 df_survival_sal_suv = df_survival_sal_suv.rename(columns={'Total':'Total_Survived'})
 df_survival_sal_suv = df_survival_sal_suv[['Salutation','Total_Survived']]
df_survival_sal_total = df_survival_sal_total.join(df_survival_sal_suv.set_index('Salutation'), on='Salutation', how='outer')
 df_survival_sal_total = df_survival_sal_total.join(df_survival_sal_nsuv.set_index('Salutation'), on='Salutation', how='outer')
 df_survival_sal_total = df_survival_sal_total.fillna(value=0)
 df_survival_sal_total
odd_ones=odd[odd==True]
odd_comments=train.loc[list(odd_ones.index)]
odd_comments[odd_comments.clean==False].count() /len(odd_comments )<filter>
ptbl_survived = pd.DataFrame.pivot_table(df_survivalinfo, values=['Fare', 'Age', 'Survived'], index=['Sex', 'Pclass'],
aggfunc={'Fare': np.mean,'Age': [min, max, np.mean], 'Survived': ['sum']})
 ptbl_survived
empty_comments=train[train.comment_text==""]
empty_comments<count_duplicates>
df_all_knn_hot = df_all.copy()
df_all_knn_hot = df_all_knn_hot.drop(['Name','Cabin','Fare','Ticket','Lastname'], axis=1)
 df_all_knn_hot = pd.get_dummies(df_all_knn_hot, columns=['Sex','Salutation','Embarked'])
 X_train, X_test, y_train, y_test = train_test_split(df_all_knn_hot.loc[train_index,:],
Survived, test_size = 0.30, random_state = 45 )
duplicate=train.comment_text.duplicated()
duplicate[duplicate==True]<filter>
myList = list(range(1,50))
neighbors = list(myList)
 cv_scores = []
for k in neighbors:
knn = KNeighborsClassifier(n_neighbors=k)
 scores = cross_val_score(knn, X_train, y_train, cv=10, scoring='accuracy')
 cv_scores.append(scores.mean() )
toxic=train[train.toxic==1]['comment_text'].values
severe_toxic=train[train.severe_toxic==1]['comment_text'].values
obscene=train[train.obscene==1]['comment_text'].values
threat=train[train.threat==1]['comment_text'].values
insult=train[train.insult==1]['comment_text'].values
identity_hate=train[train.identity_hate==1]['comment_text'].values<categorify>
X_train = X_train
y_train = y_train
KNNC = KNeighborsClassifier(n_neighbors=3)
 KNNC.fit(X_train, y_train)
 y_pred = KNNC.predict(X_test)
 print(classification_report(y_test, y_pred, target_names=['0','1']))
print("Models accuracy score: ", accuracy_score(y_test, y_pred))
replacement_patterns = [ 
(r'won't', 'will not'), 
(r'can't', 'cannot'), 
(r'i'm', 'i am'), 
(r'ain't', 'is not'), 
(r'(\w+)'ll', '\g<1> will'), 
(r'(\w+)n't', '\g<1> not'), 
(r'(\w+)'ve', '\g<1> have'), 
(r'(\w+)'s', '\g<1> is'), 
(r'(\w+)'re', '\g<1> are'), 
(r'(\w+)'d', '\g<1> would')
 ]
class RegexpReplacer(object):
def __init__(self, patterns=replacement_patterns):
self.patterns = [(re.compile(regex), repl)for(regex, repl)in
patterns]
def replace(self, text):
s = text
for(pattern, repl)in self.patterns:
s = re.sub(pattern, repl, s)
 return s<string_transform>
classes = ["will not suvive", "will survive"]
visualizer = ClassificationReport(KNNC, classes=classes, support=True)
 visualizer.fit(X_train, y_train)
 visualizer.score(X_test, y_test)
 g = visualizer.poof()
lemmer = WordNetLemmatizer()
stopwords = nltk.corpus.stopwords.words('english')
 replacer = RegexpReplacer()
tokenizer=TweetTokenizer()
def comment_process(category):
category_processed=[]
for i in range(category.shape[0]):
comment_list=tokenizer.tokenize(replacer.replace(category[i]))
comment_list_cleaned= [word for word in comment_list if(word.lower() not in stopwords
and word.lower() not in list(string.punctuation)) ]
comment_list_lemmed=[lemmer.lemmatize(word, 'v')for word in comment_list_cleaned]
category_processed.extend(list(comment_list_lemmed))
return category_processed<set_options>
titanic_submission = pd.DataFrame({'PassengerId':df_all_knn_hot.loc[test_index,:].index,
'Survived':KNNC.predict(df_all_knn_hot.loc[test_index,:])})
 titanic_submission.PassengerId = titanic_submission.PassengerId.astype(int)
 titanic_submission.Survived = titanic_submission.Survived.astype(int)
 titanic_submission.groupby('Survived' ).count()
warnings.filterwarnings('ignore' )<feature_engineering>
titanic_submission.to_csv("titanic_submission_knn_4.csv", index=False )
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text])
 word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 1),
max_features=20000)
 word_vectorizer.fit(all_text)
 train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text)
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 4),
max_features=20000)
 char_vectorizer.fit(all_text)
 train_char_features = char_vectorizer.transform(train_text)
 test_char_features = char_vectorizer.transform(test_text)
 train_features = hstack([train_char_features, train_word_features])
 test_features = hstack([test_char_features, test_word_features])
 scores = []
submission = pd.DataFrame.from_dict({'id': test['id']})
 for class_name in class_names:
train_target = train[class_name]
classifier = LogisticRegression(solver='sag')
 cv_score = np.mean(cross_val_score(classifier, train_features, train_target, cv=3, scoring='roc_auc'))
scores.append(cv_score)
 print('CV score for class {} is {}'.format(class_name, cv_score))
classifier.fit(train_features, train_target)
 submission[class_name] = classifier.predict_proba(test_features)[:, 1]
print('Total CV score is {}'.format(np.mean(scores)))
 submission.to_csv('submission.csv', index=False )<install_modules>
df_all_rf_hot = df_all.copy()
df_all_rf_hot = df_all_rf_hot.drop(['Name','Cabin','Fare','Ticket','Lastname'], axis=1)
 df_all_rf_hot = pd.get_dummies(df_all_rf_hot, columns=['Sex','Salutation','Embarked'])
 X_train, X_test, y_train, y_test = train_test_split(df_all_rf_hot.loc[train_index,:],
Survived, test_size = 0.30, random_state = 45 )
!pip install tensorflow-gpu==2.0a0<import_modules>
myList = list(range(1,30))
levels = list(myList)
 cv_scores = []
for l in levels:
rfc = RandomForestClassifier(n_estimators=100)
 scores = cross_val_score(rfc, X_train, y_train, cv=5, scoring='recall')
 cv_scores.append(scores.mean() )
print(tf.__version__ )<set_options>
X_train = X_train
y_train = y_train
RFCC = RandomForestClassifier(max_depth=14,
n_estimators=5000)
 RFCC.fit(X_train, y_train)
 y_pred = RFCC.predict(X_test)
 print(classification_report(y_test, y_pred, target_names=['0','1']))
print("Models accuracy score: ", accuracy_score(y_test, y_pred))
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
<define_variables>
classes = ["will not suvive", "will survive"]
visualizer = ClassificationReport(RFCC, classes=classes, support=True)
 visualizer.fit(X_train, y_train)
 visualizer.score(X_test, y_test)
 g = visualizer.poof()
tf.random.set_seed(42)
 datadir = ".. /input/"<choose_model_class>
titanic_submission_rfc = pd.DataFrame({'PassengerId':df_all_rf_hot.loc[test_index,:].index,
'Survived':RFCC.predict(df_all_rf_hot.loc[test_index,:])})
 titanic_submission_rfc.PassengerId = titanic_submission_rfc.PassengerId.astype(int)
 titanic_submission_rfc.Survived = titanic_submission_rfc.Survived.astype(int)
 titanic_submission_rfc.groupby('Survived' ).count()
class Message_Passer_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Message_Passer_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )<normalization>
titanic_submission_rfc.to_csv("titanic_submission_rfc_5.csv", index=False )
class Message_Agg(tf.keras.layers.Layer):
def __init__(self):
super(Message_Agg, self ).__init__()
def call(self, messages):
return tf.math.reduce_sum(messages, 2 )<choose_model_class>
import scipy.special
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
<choose_model_class>
conf_performance_list = []
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )<choose_model_class>
input_nodes = 29
hidden_nodes = 3
output_nodes = 2
learningrate = 0.4
nn_epochs = 1000
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
<choose_model_class>
titanic_nn = neuralNetwork(input_nodes, hidden_nodes, output_nodes, learningrate )
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n = tf.keras.layers.BatchNormalization()
self.batch_norm_e = tf.keras.layers.BatchNormalization()
def call(self, nodes, edges, mask):
nodes_0 = nodes
edges_0 = edges
n_nodes = tf.shape(nodes_0)[1]
node_dim = tf.shape(nodes_0)[2]
state_i = tf.tile(nodes_0, [1, n_nodes, 1])
 state_j = tf.reshape(tf.tile(nodes_0, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges = self.adj_updaters(state_i, state_j, edges_0)
 new_edges = tf.math.multiply(new_edges, mask)
 messages = self.message_passers(state_i, state_j, new_edges)
 masked = tf.math.multiply(messages, mask)
 masked = tf.reshape(masked, [tf.shape(messages)[0], tf.shape(nodes_0)[1], tf.shape(nodes_0)[1], tf.shape(messages)[2]])
 agg_m = self.message_aggs(masked)
 state_1 = self.update_functions(nodes_0, agg_m)
 nodes_out = self.batch_norm_n(state_1)
 edges_out = self.batch_norm_e(new_edges)
 return nodes_out, edges_out<choose_model_class>
X_train_nn, X_test_nn, y_train_nn, y_test_nn = train_test_split(df_all_knn_hot.loc[train_index,:], Survived, test_size = 0.15, random_state = 45 )
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes, edges, mask):
nodes_0 = nodes
edges_0 = edges
n_nodes = tf.shape(nodes_0)[1]
node_dim = tf.shape(nodes_0)[2]
state_i = tf.tile(nodes_0, [1, n_nodes, 1])
 state_j = tf.reshape(tf.tile(nodes_0, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges = self.adj_updaters(state_i, state_j, edges_0)
 new_edges = tf.math.multiply(new_edges, mask)
 edges_out = new_edges
return edges_out<choose_model_class>
Xy_train_nn = pd.concat([X_train_nn, y_train_nn], axis=1 )
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = self.embed_node(nodes)
 edges = self.embed_edge(edges)
 nodes_ = nodes
edges_ = edges
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes, edges = mp(nodes, edges, mask)
 nodes = nodes - nodes_
edges = edges - edges_
nodes_ = nodes
edges_ = edges
else:
nodes, edges = mp(nodes, edges, mask)
 edges = self.MP_edge(nodes, edges, mask)
 con_edges = self.edge_regressor(edges)
 return con_edges<compute_test_metric>
epochs = nn_epochs
for e in range(epochs):
for row in Xy_train_nn.itertuples() :
inputs =(np.asfarray(row[1:30])/ 29 * 0.99)+ 0.01
targets = np.zeros(output_nodes)+ 0.01
targets[int(row[30])] = 0.99
titanic_nn.train(inputs, targets)
 pass
pass
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error<choose_model_class>
Xy_test_nn = pd.concat([X_test_nn, y_test_nn], axis=1 )
learning_rate = 0.001
def warmup(epoch):
initial_lrate = learning_rate
if epoch == 0:
lrate = 0.00001
if epoch == 1:
lrate = 0.0001
if epoch > 1:
lrate = 0.001
if epoch > 20:
lrate = 0.0001
if epoch > 25:
lrate = 0.00001
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(warmup)
 opt = tf.optimizers.Adam(learning_rate=learning_rate)
<choose_model_class>
scorecard = []
matrixlist = []
for index,row in Xy_test_nn.iterrows() :
inputs = row[0:29].values
correct_label = row[29]
results = titanic_nn.query(inputs)
 label = np.argmax(results)
 print('PassengerID:', index, ' - Networks answer: ', label, ' --> Correct answer: ', correct_label)
 matrixlist.append([results, label, correct_label])
 if(label == correct_label):
scorecard.append(1)
 else:
scorecard.append(0)
 pass
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 7)
 mpnn.compile(opt, log_mae )<define_variables>
scorecard_array = np.array(scorecard)
 nn_accuracy = scorecard_array.sum() / scorecard_array.size
conf_performance_list.append([nn_accuracy,learningrate,hidden_nodes,epochs])
 print('Accuracy score by "75/15"-network: ', nn_accuracy )
batch_size = 64
epochs = 30 
<train_model>
epochs = nn_epochs
for e in range(epochs):
for row in Xy_test_nn.itertuples() :
inputs =(np.asfarray(row[1:30])/ 29 * 0.99)+ 0.01
targets = np.zeros(output_nodes)+ 0.01
targets[int(row[30])] = 0.99
titanic_nn.train(inputs, targets)
 pass
pass
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size<train_model>
titanic_submission_nn = pd.DataFrame(columns=['PassengerId','Survived'])
 for index,row in df_all_rf_hot.loc[test_index].iterrows() :
inputs = row[0:29].values
results = titanic_nn.query(inputs)
 label = np.argmax(results)
 titanic_submission_nn = titanic_submission_nn.append({'PassengerId' : index , 'Survived': label} , ignore_index=True)
 pass
preds, train_size = train()<load_pretrained>
titanic_submission_nn.PassengerId = titanic_submission_nn.PassengerId.astype(int)
 titanic_submission_nn.Survived = titanic_submission_nn.Survived.astype(int)
 titanic_submission_nn.groupby('Survived' ).count()
mpnn.save_weights("mymodel.h5" )<load_from_csv>
titanic_submission_nn.to_csv("titanic_submission_nn_6.csv", index=False )
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )<compute_test_metric>
import tensorflow as tf
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()<normalization>
X_train_tfnn, X_test_tfnn, y_train_tfnn, y_test_tfnn = train_test_split(df_all_knn_hot.loc[train_index,:], Survived, test_size = 0.15, random_state = 45)
 X_train_tfnn = tf.keras.utils.normalize(np.asfarray(X_train_tfnn),axis= -1)
 X_test_tfnn = tf.keras.utils.normalize(np.asfarray(X_test_tfnn),axis= -1)
 y_train_tfnn = np.asfarray(y_train_tfnn)
 y_test_tfnn = np.asfarray(y_test_tfnn )
max_size = 29
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(val_group, preds )<feature_engineering>
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(2, activation=tf.nn.softmax))
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.fit(X_train_tfnn, y_train_tfnn, epochs = 200 )
val['pred_scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]*scale_norm + scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]<compute_train_metric>
val_loss, val_acc = model.evaluate(X_test_tfnn, y_test_tfnn )
for coup in coups_to_isolate:
log_mae = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'][val.type == coup])
 print(coup,"\t", log_mae)
 total = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'])
 print("")
 print("Total:","\t", total )<load_pretrained>
model.save('titanic_survivor_predictor.model' )
nodes_test = np.load(datadir + "internalgraphdata/nodes_test.npz")['arr_0']
in_edges_test = np.load(datadir + "internalgraphdata/in_edges_test.npz")['arr_0']
in_edges_test = in_edges_test.reshape(-1,in_edges_test.shape[1]*in_edges_test.shape[2],in_edges_test.shape[3] )<predict_on_test>
new_model = tf.keras.models.load_model('titanic_survivor_predictor.model' )