Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<define_variables>
!pip install tensorflow-gpu==2.0a0
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
EMBEDDING_DIM = 300
all_words = set(cv1.vocabulary_.keys() ).union(set(cv2.vocabulary_.keys())).union(set(cv3.vocabulary_.keys())) 
<categorify>
print(tf.__version__ )
def get_embedding() :
embeddings_index = {}
f = open(EMBEDDING_FILE)
 for line in f:
values = line.split()
word = values[0]
if len(values)== EMBEDDING_DIM + 1 and word in all_words:
coefs = np.asarray(values[1:], dtype="float32")
 embeddings_index[word] = coefs
f.close()
return embeddings_index
embeddings_index = get_embedding()
print("Number of words that don't exist in GLOVE:", len(all_words - set(embeddings_index)) )<define_variables>
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
MAX_SEQUENCE_LENGTH = 200<feature_engineering>
datadir = ".. /input/"
nodes_train = np.load(datadir + "champs-basic-graph/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "champs-basic-graph/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "champs-basic-graph/out_edges_train.npz")['arr_0']
nodes_test = np.load(datadir + "champs-basic-graph/nodes_test.npz")['arr_0']
in_edges_test = np.load(datadir + "champs-basic-graph/in_edges_test.npz")['arr_0']
tokenizer = Tokenizer()
np_text = np.append(x_train_text.values,x_val_text.values)
 tokenizer.fit_on_texts(np.append(np_text, test['clean_text'].values))
word_index = tokenizer.word_index
nb_words = len(word_index)+ 1
embedding_matrix = np.random.rand(nb_words, EMBEDDING_DIM + 2)
 for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 sent = textblob.TextBlob(word ).sentiment
if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, [sent.polarity, sent.subjectivity])
 else:
embedding_matrix[i, -2:] = [sent.polarity, sent.subjectivity]
train_seq = pad_sequences(tokenizer.texts_to_sequences(x_train_text), maxlen=MAX_SEQUENCE_LENGTH)
 val_seq = pad_sequences(tokenizer.texts_to_sequences(x_val_text), maxlen=MAX_SEQUENCE_LENGTH)
 test_seq = pad_sequences(tokenizer.texts_to_sequences(test['clean_text']), maxlen=MAX_SEQUENCE_LENGTH )<choose_model_class>
nodes_train, in_edges_train, out_labels = shuffle(nodes_train, in_edges_train, out_labels )
def build_model() :
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,))
x = Embedding(nb_words, EMBEDDING_DIM + 2, weights=[embedding_matrix],trainable = True )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 dropout = SpatialDropout1D(0.2)
 dense_input = Input(shape=(100,))
dense_vector = BatchNormalization()(dense_input)
 feature_vector = concatenate([x, dense_vector])
 feature_vector = Dense(128, activation="relu" )(feature_vector)
 output = Dense(6, activation="sigmoid" )(feature_vector)
 model = Model(inputs=[sequence_input, dense_input], outputs=output)
 return model<train_model>
class Message_Passer_NNM(tf.keras.layers.Layer):
def __init__(self, node_dim):
super(Message_Passer_NNM, self ).__init__()
self.node_dim = node_dim
self.nn = tf.keras.layers.Dense(units=self.node_dim*self.node_dim, activation = tf.nn.relu)
 def call(self, node_j, edge_ij):
A = self.nn(edge_ij)
 A = tf.reshape(A, [-1, self.node_dim, self.node_dim])
 node_j = tf.reshape(node_j, [-1, self.node_dim, 1])
 messages = tf.linalg.matmul(A, node_j)
 messages = tf.reshape(messages, [-1, tf.shape(edge_ij)[1], self.node_dim])
 return messages
model = build_model()
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
 early_stopping = EarlyStopping(monitor="val_acc", patience=2, verbose=1)
 print("Training the model")
 model.fit([train_seq, x_train_vec], y_train.values, validation_data=([val_seq, x_val_vec], y_val.values),epochs=2,
batch_size=1024, shuffle=True, callbacks=[early_stopping], verbose=1 )<predict_on_test>
class Message_Agg(tf.keras.layers.Layer):
def __init__(self):
super(Message_Agg, self ).__init__()
def call(self, messages):
return tf.math.reduce_sum(messages, 2 )
test_preds = model.predict([test_seq, x_test_tfidf_vec], batch_size=1024, verbose=1 )<define_variables>
class Update_Func_GRU(tf.keras.layers.Layer):
def __init__(self, state_dim):
super(Update_Func_GRU, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate(axis=1)
 self.GRU = tf.keras.layers.GRU(state_dim)
 def call(self, old_state, agg_messages):
n_nodes = tf.shape(old_state)[1]
node_dim = tf.shape(old_state)[2]
old_state = tf.reshape(old_state, [-1, 1, tf.shape(old_state)[-1]])
 agg_messages = tf.reshape(agg_messages, [-1, 1, tf.shape(agg_messages)[-1]])
 concat = self.concat_layer([old_state, agg_messages])
 activation = self.GRU(concat)
 activation = tf.reshape(activation, [-1, n_nodes, node_dim])
 return activation
labels = ['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']<create_dataframe>
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1, activation=None)
 def call(self, nodes, edges):
n_nodes = tf.shape(nodes)[1]
node_dim = tf.shape(nodes)[2]
state_i = tf.reshape(tf.tile(nodes, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 state_j = tf.tile(nodes, [1, n_nodes, 1])
 concat = self.concat_layer([state_i, edges, state_j])
 activation_1 = self.hidden_layer_1(concat)
 activation_2 = self.hidden_layer_2(activation_1)
 return self.output_layer(activation_2 )
y_df = np.where(test_preds > 0.5, 1, 0)
 y_df = pd.DataFrame(y_df, columns=['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])
 y_df = y_df.astype('int' )<compute_test_metric>
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, state_dim):
super(MP_Layer, self ).__init__(self)
 self.message_passers = Message_Passer_NNM(node_dim = state_dim)
 self.message_aggs = Message_Agg()
self.update_functions = Update_Func_GRU(state_dim = state_dim)
 self.state_dim = state_dim
def call(self, nodes, edges, mask):
n_nodes = tf.shape(nodes)[1]
node_dim = tf.shape(nodes)[2]
state_j = tf.tile(nodes, [1, n_nodes, 1])
 messages = self.message_passers(state_j, edges)
 masked = tf.math.multiply(messages, mask)
 masked = tf.reshape(masked, [tf.shape(messages)[0], n_nodes, n_nodes, node_dim])
 agg_m = self.message_aggs(masked)
 updated_nodes = self.update_functions(nodes, agg_m)
 nodes_out = updated_nodes
return nodes_out
def get_metri_scores(y_test, y_test_pred):
vals = precision_recall_fscore_support(y_test, y_test_pred, average='macro')
 precision = vals[0]
recall = vals[1]
f1 = vals[2]
acc = accuracy_score(y_test, y_test_pred)
 return precision, recall, f1, acc<feature_engineering>
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.T = T
self.embed = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.MP = MP_Layer(state_dim)
 self.edge_regressor = Edge_Regressor(out_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = self.embed(nodes)
 for mp in range(self.T):
nodes = self.MP(nodes, edges, mask)
 con_edges = self.edge_regressor(nodes, edges)
 return con_edges
results_cv = pd.DataFrame({'labels': labels})
 results_cv['acc'] = 0
results_cv['f1'] = 0
results_cv['precision'] = 0
results_cv['recall'] = 0
for col in labels:
print(col)
 precision, recall, f1, acc = get_metri_scores(test[col], y_df[col])
 results_cv['acc'][results_cv['labels']==col] = acc
results_cv['f1'][results_cv['labels']==col] = f1
results_cv['precision'][results_cv['labels']==col] = precision
results_cv['recall'][results_cv['labels']==col] = recall<set_options>
def mse(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.square(tf.subtract(nums, preds)))
 return reconstruction_error
%matplotlib inline 
<define_variables>
def log_mse(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.square(tf.subtract(nums, preds))))
return reconstruction_error
maxlen = 100
max_features = 100000
embed_size = 300
embedding_path = ".. /input/glove840b300dtxt/glove.840B.300d.txt"
patience = 5
n_epochs = 50
batch_size = 512
seed=1029<load_from_csv>
def mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.abs(tf.subtract(nums, preds)))
 return reconstruction_error
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 subm = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 train_test_boundary = len(train['id'])
 df = pd.concat([train.drop(['id','toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'],axis=1),test.drop('id',axis=1)])
 label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train['none'] = 1-train[label_cols].max(axis=1)
 train['multi'] = train[label_cols].sum(axis=1)
 df['lowered_comment'] = df['comment_text'].apply(lambda x: x.lower())
 contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
df['treated_question'] = df['lowered_comment'].apply(lambda x: clean_contractions(x, contraction_mapping))
def clean_text(x):
x = str(x)
 for punct in "/-'—":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
df['treated_question'] = df['lowered_comment'].apply(lambda x: clean_text(x))
mispell_dict = {"youfuck":"you fuck","niggors":"niggers","néger":'niger',"fucksex":"fuck sex","yourselfgo":"yourself go","bitchbot":"bitch bot","donkeysex":"donkey sex","mothjer":"mother","niggerjew":"nigger jew","gayyour":"gay your","motherfuckerdie":"motherfucker die","radicalnigger":"radical nigger","philippineslong":"philippines long",'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
df['treated_comment'] = df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
list_classes = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
y_train = train[list_classes].values
list_sentences_train = df["treated_comment"].iloc[:train_test_boundary]
list_sentences_test = df["treated_comment"].iloc[train_test_boundary:]
del df
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_train = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_test = pad_sequences(list_tokenized_test, maxlen=maxlen)
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))
all_embs = np.stack(embedding_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<train_model>
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss<choose_model_class>
learning_rate = 0.001
def step_decay(epoch):
initial_lrate = learning_rate
drop = 0.1
epochs_drop = 20.0
lrate = initial_lrate * np.power(drop,
np.floor(( epoch)/epochs_drop))
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(step_decay)
 stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 15, restore_best_weights=True)
 opt = tf.optimizers.Adam(learning_rate=learning_rate)
class CNN_Text(nn.Module):
def __init__(self, args):
super(CNN_Text, self ).__init__()
self.args = args
V = args.embed_num
D = args.embed_dim
C = args.class_num
Ci = 1
Co = args.kernel_num
Ks = args.kernel_sizes
if args.max_norm is not None:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(V, D, max_norm=5, scale_grad_by_freq=True)
 else:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(V, D, scale_grad_by_freq=True)
 if args.pre_word_Embedding:
self.embed.weight.data.copy_(torch.tensor(embedding_matrix))
self.embed.weight.requires_grad = True
print("dddd {} ".format(self.embed.weight.data.size()))
if args.wide_conv is True:
print("using wide convolution")
 self.convs1 = [nn.Conv2d(in_channels=Ci, out_channels=Co, kernel_size=(K, D), stride=(1, 1),
padding=(K//2, 0), dilation=1, bias=False)for K in Ks]
else:
print("using narrow convolution")
 self.convs1 = [nn.Conv2d(in_channels=Ci, out_channels=Co, kernel_size=(K, D), bias=True)for K in Ks]
print(self.convs1)
 if args.init_weight:
print("Initing W.......")
 for conv in self.convs1:
torch.nn.init.xavier_normal_(conv.weight.data, gain=np.sqrt(args.init_weight_value))
fan_in, fan_out = CNN_Text.calculate_fan_in_and_fan_out(conv.weight.data)
 print(" in {} out {} ".format(fan_in, fan_out))
std = np.sqrt(args.init_weight_value)* np.sqrt(2.0 /(fan_in + fan_out))
if args.cuda is True:
for conv in self.convs1:
conv = conv.cuda()
self.dropout = nn.Dropout(args.dropout)
 self.dropout_embed = nn.Dropout(args.dropout_embed)
 in_fea = len(Ks)* Co
self.fc = nn.Linear(in_features=in_fea, out_features=C, bias=True)
 if args.batch_normalizations is True:
print("using batch_normalizations in the model...... ")
 self.fc1 = nn.Linear(in_features=in_fea, out_features=in_fea//2, bias=True)
 self.fc2 = nn.Linear(in_features=in_fea//2, out_features=C, bias=True)
 self.convs1_bn = nn.BatchNorm2d(num_features=Co, momentum=args.bath_norm_momentum,
affine=args.batch_norm_affine)
 self.fc1_bn = nn.BatchNorm1d(num_features=in_fea//2, momentum=args.bath_norm_momentum,
affine=args.batch_norm_affine)
 self.fc2_bn = nn.BatchNorm1d(num_features=C, momentum=args.bath_norm_momentum,
affine=args.batch_norm_affine)
 def calculate_fan_in_and_fan_out(tensor):
dimensions = tensor.ndimension()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with less than 2 dimensions")
 if dimensions == 2:
fan_in = tensor.size(1)
 fan_out = tensor.size(0)
 else:
num_input_fmaps = tensor.size(1)
 num_output_fmaps = tensor.size(0)
 receptive_field_size = 1
if tensor.dim() > 2:
receptive_field_size = tensor[0][0].numel()
fan_in = num_input_fmaps * receptive_field_size
fan_out = num_output_fmaps * receptive_field_size
return fan_in, fan_out
def forward(self, x):
x = self.embed(x)
 x = self.dropout_embed(x)
 x = x.unsqueeze(1)
 if self.args.batch_normalizations is True:
x = [self.convs1_bn(torch.tanh(conv(x)) ).squeeze(3)for conv in self.convs1]
x = [F.max_pool1d(i, i.size(2)).squeeze(2)for i in x]
else:
x = [F.elu(conv(x)).squeeze(3)for conv in self.convs1]
x = [F.max_pool1d(i, i.size(2)).squeeze(2)for i in x]
x = torch.cat(x, 1)
 x = self.dropout(x)
 if self.args.batch_normalizations is True:
x = self.fc1_bn(self.fc1(x))
logit = self.fc2_bn(self.fc2(torch.tanh(x)))
 else:
logit = self.fc(x)
 return logit
def sigmoid(x):
return 1 /(1 + np.exp(-x))<choose_model_class>
mpnn = MPNN(out_int_dim = 512, state_dim = 128, T = 4)
 mpnn.compile(opt, log_mae, metrics = [mae, log_mse] )
class RCNN_Text(nn.Module):
def __init__(self, args):
super(RCNN_Text, self ).__init__()
self.args = args
self.output_size = args.output_size
self.hidden_size = args.hidden_size
self.vocab_size = args.vocab_size
self.embedding_size = args.embedding_size
if args.max_norm is not None:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(self.vocab_size, self.embedding_size, max_norm=5, scale_grad_by_freq=True)
 else:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(self.vocab_size, self.embedding_size, scale_grad_by_freq=True)
 if args.pre_word_Embedding:
self.embed.weight.data.copy_(torch.tensor(embedding_matrix))
self.embed.weight.requires_grad = True
print("dddd {} ".format(self.embed.weight.data.size()))
self.dropout = nn.Dropout(args.dropout)
 self.dropout_embed = nn.Dropout(args.dropout_embed)
 self.lstm = nn.LSTM(self.embedding_size, self.hidden_size, bidirectional=True)
 self.W2 = nn.Linear(2 * self.hidden_size + self.embedding_size, self.hidden_size)
 self.label = nn.Linear(self.hidden_size, self.output_size)
 def forward(self,x):
x = self.embed(x)
 x = self.dropout_embed(x)
 batch_size = x.size(0)
 x = x.permute(1, 0, 2)
 h_0 = Variable(torch.zeros(2, batch_size, self.hidden_size ).cuda())
 c_0 = Variable(torch.zeros(2, batch_size, self.hidden_size ).cuda())
 output,(final_hidden_state, final_cell_state)= self.lstm(x,(h_0, c_0))
final_encoding = torch.cat(( output, x), 2 ).permute(1, 0, 2)
 y = self.W2(final_encoding)
 y = y.permute(0, 2, 1)
 y = F.max_pool1d(y, y.size() [2])
 y = y.squeeze(2)
 logits = self.label(y)
 return logits 
<init_hyperparams>
train_size = int(len(out_labels)*0.8)
 batch_size = 16
epochs = 25
class RNN_selfatten(nn.Module):
def __init__(self,args):
super(RNN_selfatten, self ).__init__()
self.args = args
self.output_size = args.output_size
self.hidden_size = args.hidden_size
self.vocab_size = args.vocab_size
self.embedding_size = args.embedding_size
da = 350
r = 30
self.W_s1 = nn.Linear(2*self.hidden_size, da)
 self.W_s2 = nn.Linear(da, r)
 self.fc_layer = nn.Linear(r*2*self.hidden_size, 2000)
 self.label = nn.Linear(2000, self.output_size)
 if args.max_norm is not None:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(self.vocab_size, self.embedding_size, max_norm=5, scale_grad_by_freq=True)
 else:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(self.vocab_size, self.embedding_size, scale_grad_by_freq=True)
 if args.pre_word_Embedding:
self.embed.weight.data.copy_(torch.tensor(embedding_matrix))
self.embed.weight.requires_grad = True
print("dddd {} ".format(self.embed.weight.data.size()))
self.dropout = nn.Dropout(args.dropout)
 self.dropout_embed = nn.Dropout(args.dropout_embed)
 self.bilstm = nn.LSTM(self.embedding_size, self.hidden_size, bidirectional=True)
 def attention_net(self, lstm_output):
attn_weight_matrix = self.W_s2(torch.tanh(self.W_s1(lstm_output)))
 attn_weight_matrix = attn_weight_matrix.permute(0, 2, 1)
 attn_weight_matrix = F.softmax(attn_weight_matrix, dim=2)
 return attn_weight_matrix
def forward(self,x):
x = self.embed(x)
 x = self.dropout_embed(x)
 batch_size = x.size(0)
 x = x.permute(1, 0, 2)
 h_0 = Variable(torch.zeros(2, batch_size, self.hidden_size ).cuda())
 c_0 = Variable(torch.zeros(2, batch_size, self.hidden_size ).cuda())
 output,(h_n, c_n)= self.bilstm(x,(h_0, c_0))
output = output.permute(1, 0, 2)
 attn_weight_matrix = self.attention_net(output)
 hidden_matrix = torch.bmm(attn_weight_matrix, output)
 fc_out = self.fc_layer(hidden_matrix.view(-1, hidden_matrix.size() [1]*hidden_matrix.size() [2]))
logits = self.label(fc_out)
 return logits<choose_model_class>
mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]} )
class RNN_atten(nn.Module):
def __init__(self,args):
super(RNN_atten, self ).__init__()
self.args = args
self.output_size = args.output_size
self.hidden_size = args.hidden_size
self.vocab_size = args.vocab_size
self.embedding_size = args.embedding_size
if args.max_norm is not None:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(self.vocab_size, self.embedding_size, max_norm=5, scale_grad_by_freq=True)
 else:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(self.vocab_size, self.embedding_size, scale_grad_by_freq=True)
 if args.pre_word_Embedding:
self.embed.weight.data.copy_(torch.tensor(embedding_matrix))
self.embed.weight.requires_grad = True
print("dddd {} ".format(self.embed.weight.data.size()))
self.dropout = nn.Dropout(args.dropout)
 self.dropout_embed = nn.Dropout(args.dropout_embed)
 self.label = nn.Linear(self.hidden_size, self.output_size)
 self.lstm = nn.LSTM(self.embedding_size, self.hidden_size, bidirectional=False)
 def attention_net(self, lstm_output, final_state):

 hidden = final_state.squeeze(0)
 attn_weights = torch.bmm(lstm_output, hidden.unsqueeze(2)).squeeze(2)
 soft_attn_weights = F.softmax(attn_weights, 1)
 new_hidden_state = torch.bmm(lstm_output.transpose(1, 2), soft_attn_weights.unsqueeze(2)).squeeze(2)
 return new_hidden_state
def forward(self,x):
x = self.embed(x)
 x = self.dropout_embed(x)
 batch_size = x.size(0)
 x = x.permute(1, 0, 2)
 h_0 = Variable(torch.zeros(1, batch_size, self.hidden_size ).cuda())
 c_0 = Variable(torch.zeros(1, batch_size, self.hidden_size ).cuda())
 output,(final_hidden_state, final_cell_state)= self.lstm(x,(h_0, c_0))
output = output.permute(1, 0, 2)
 attn_output = self.attention_net(output, final_hidden_state)
 logits = self.label(attn_output)
 return logits<define_variables>
mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate, stop_early], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
class CharCNNDictionary:
def __init__(self):
self.ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'"/\\|_@
'
self.PAD_TOKEN = '<PAD>'
def build_dictionary(self):
self.vocab_chars = list(self.ALPHABET)+ [self.PAD_TOKEN]
self.char2idx = {char:idx for idx, char in enumerate(self.vocab_chars)}
self.vocabulary_size = len(self.vocab_chars)
 self._build_weight()
def _build_weight(self):
onehot_matrix = np.eye(self.vocabulary_size, self.vocabulary_size - 1)
 self.embedding = onehot_matrix
def indexer(self, char):
try:
return self.char2idx[char]
except:
char = self.PAD_TOKEN
return self.char2idx[char]
def pad_text(text, pad, min_length=None, max_length=None):
length = len(text)
 if min_length is not None and length < min_length:
return text + [pad]*(min_length - length)
 if max_length is not None and length > max_length:
return text[:max_length]
return text
class TextDataset(Dataset):
def __init__(self, texts, dictionary, sort=False, min_length=None, max_length=None):
PAD_IDX = dictionary.indexer(dictionary.PAD_TOKEN)
 self.texts = [[dictionary.indexer(token)for token in text]
for text in texts]
if min_length or max_length:
self.texts = [pad_text(text, PAD_IDX, min_length, max_length)
 for text in self.texts]
if sort:
self.texts = sorted(self.texts, key=lambda x: len(x[0]))
def get_all(self):
return self.texts
def __getitem__(self, index):
tokens = self.texts[index]
return tokens
def __len__(self):
return len(self.texts )<init_hyperparams>
preds = mpnn.predict({'adj_input' : in_edges_test, 'nod_input': nodes_test} )
class char_CNN(nn.Module):
def __init__(self,args):
super(char_CNN, self ).__init__()
self.args = args
vocabulary_size = args.vocabulary_size
embed_size = vocabulary_size - 1
embedding_weight = args.embedding
n_classes = args.output_size
max_length = args.max_length
if args.mode == 'large':
conv_features = 1024
linear_features = 2048
elif args.mode == 'small':
conv_features = 256
linear_features = 1024
else:
raise NotImplementedError()
self.embedding = nn.Embedding(vocabulary_size, embed_size)
 if embedding_weight is not None:
self.embedding.weight = nn.Parameter(torch.FloatTensor(embedding_weight), requires_grad=False)
 conv1 = nn.Sequential(
 nn.Conv1d(in_channels=embed_size, out_channels=conv_features, kernel_size=7),
nn.MaxPool1d(kernel_size=3),
nn.ReLU() 
)
 conv2 = nn.Sequential(
 nn.Conv1d(in_channels=conv_features, out_channels=conv_features, kernel_size=7),
nn.MaxPool1d(kernel_size=3),
nn.ReLU() 
)
 conv3 = nn.Sequential(
 nn.Conv1d(in_channels=conv_features, out_channels=conv_features, kernel_size=3),
nn.ReLU() 
)
 conv4 = nn.Sequential(
 nn.Conv1d(in_channels=conv_features, out_channels=conv_features, kernel_size=3),
nn.ReLU() 
)
 conv5 = nn.Sequential(
 nn.Conv1d(in_channels=conv_features, out_channels=conv_features, kernel_size=3),
nn.ReLU() 
)
 conv6 = nn.Sequential(
 nn.Conv1d(in_channels=conv_features, out_channels=conv_features, kernel_size=3),
nn.MaxPool1d(kernel_size=3),
nn.ReLU() 
)
 initial_linear_size =(max_length - 96)// 27 * conv_features
linear1 = nn.Sequential(
 nn.Linear(initial_linear_size, linear_features),
nn.Dropout() ,
nn.ReLU() 
)
 linear2 = nn.Sequential(
 nn.Linear(linear_features, linear_features),
nn.Dropout() ,
nn.ReLU() 
)
 linear3 = nn.Linear(linear_features, n_classes)
 self.convolution_layers = nn.Sequential(conv1, conv2, conv3, conv4, conv5, conv6)
 self.linear_layers = nn.Sequential(linear1, linear2, linear3)
 self.dropout = nn.Dropout(args.dropout)
 self.dropout_embed = nn.Dropout(args.dropout_embed)
 def forward(self, sentences):
x = self.embedding(sentences)
 x = x.transpose(1,2)
 x = self.convolution_layers(x)
 x = x.view(x.size(0), -1)
 x = self.linear_layers(x)
 return x<choose_model_class>
np.save("preds_kernel.npy" , preds )
class C_LSTM(nn.Module):
def __init__(self, args):
super(C_LSTM, self ).__init__()
self.args = args
V = args.embed_num
D = args.embed_dim
C = args.class_num
Ci = 1
Co = args.kernel_num
Ks = args.kernel_sizes
self.hidden_size = args.hidden_size
if args.max_norm is not None:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(V, D, max_norm=5, scale_grad_by_freq=True)
 else:
print("max_norm = {} ".format(args.max_norm))
self.embed = nn.Embedding(V, D, scale_grad_by_freq=True)
 if args.pre_word_Embedding:
self.embed.weight.data.copy_(torch.tensor(embedding_matrix))
self.embed.weight.requires_grad = True
print("dddd {} ".format(self.embed.weight.data.size()))
if args.wide_conv is True:
print("using wide convolution")
 self.convs1 = [nn.Conv2d(in_channels=Ci, out_channels=Co, kernel_size=(K, D), stride=(1, 1),
padding=(K//2, 0), dilation=1, bias=False)for K in Ks]
else:
print("using narrow convolution")
 self.convs1 = [nn.Conv2d(in_channels=Ci, out_channels=Co, kernel_size=(K, D), bias=True)for K in Ks]
print(self.convs1)
 if args.init_weight:
print("Initing W.......")
 for conv in self.convs1:
torch.nn.init.xavier_normal_(conv.weight.data, gain=np.sqrt(args.init_weight_value))
fan_in, fan_out = C_LSTM.calculate_fan_in_and_fan_out(conv.weight.data)
 print(" in {} out {} ".format(fan_in, fan_out))
std = np.sqrt(args.init_weight_value)* np.sqrt(2.0 /(fan_in + fan_out))
if args.cuda is True:
for conv in self.convs1:
conv = conv.cuda()
self.dropout = nn.Dropout(args.dropout)
 self.dropout_embed = nn.Dropout(args.dropout_embed)
 self.lstm = nn.LSTM(Co, self.hidden_size, bidirectional=True)
 in_fea = 2 * self.hidden_size
self.fc = nn.Linear(in_features=in_fea, out_features=C, bias=True)
 if args.batch_normalizations:
print("using batch_normalizations in the model...... ")
 self.fc1 = nn.Linear(in_features=in_fea, out_features=in_fea//2, bias=True)
 self.fc2 = nn.Linear(in_features=in_fea//2, out_features=C, bias=True)
 self.convs1_bn = nn.BatchNorm2d(num_features=Co, momentum=args.bath_norm_momentum,
affine=args.batch_norm_affine)
 self.fc1_bn = nn.BatchNorm1d(num_features=in_fea//2, momentum=args.bath_norm_momentum,
affine=args.batch_norm_affine)
 self.fc2_bn = nn.BatchNorm1d(num_features=C, momentum=args.bath_norm_momentum,
affine=args.batch_norm_affine)
 def calculate_fan_in_and_fan_out(tensor):
dimensions = tensor.ndimension()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with less than 2 dimensions")
 if dimensions == 2:
fan_in = tensor.size(1)
 fan_out = tensor.size(0)
 else:
num_input_fmaps = tensor.size(1)
 num_output_fmaps = tensor.size(0)
 receptive_field_size = 1
if tensor.dim() > 2:
receptive_field_size = tensor[0][0].numel()
fan_in = num_input_fmaps * receptive_field_size
fan_out = num_output_fmaps * receptive_field_size
return fan_in, fan_out
def forward(self, x):
x = self.embed(x)
 x = self.dropout_embed(x)
 x = x.unsqueeze(1)
 batch_size = x.size(0)
 if self.args.batch_normalizations:
x = [self.convs1_bn(torch.tanh(conv(x)) ).squeeze(3)for conv in self.convs1]
x = [i.permute(2, 0,1)for i in x]
else:
x = [F.relu(conv(x)).squeeze(3)for conv in self.convs1]
x = [i.permute(2, 0,1)for i in x]
x = x[0]
h_0 = Variable(torch.zeros(2, batch_size, self.hidden_size ).cuda())
 c_0 = Variable(torch.zeros(2, batch_size, self.hidden_size ).cuda())
 output,(final_hidden_state, final_cell_state)= self.lstm(x,(h_0, c_0))
x = torch.cat([final_hidden_state[i] for i in range(final_hidden_state.size(0)) ],1)
 if self.args.batch_normalizations:
x = self.fc1_bn(self.fc1(x))
logit = self.fc2_bn(self.fc2(torch.tanh(x)))
 else:
logit = self.fc(x)
 return logit 
<import_modules>
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 test_group = test.groupby('molecule_name')
 scale_min = train['scalar_coupling_constant'].min()
scale_max = train['scalar_coupling_constant'].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
def gelu(x):
return 0.5 * x *(1 + torch.tanh(math.sqrt(2 / math.pi)*(x + 0.044715 * torch.pow(x, 3))))
def swish(x):
return x * torch.sigmoid(x)
 ACT_FNS = {
'relu': nn.ReLU,
'swish': swish,
'gelu': gelu
}
class LayerNorm(nn.Module):
"Construct a layernorm module in the OpenAI style(epsilon inside the square root )."
def __init__(self, n_state, e=1e-5):
super(LayerNorm, self ).__init__()
self.g = nn.Parameter(torch.ones(n_state))
self.b = nn.Parameter(torch.zeros(n_state))
self.e = e
def forward(self, x):
u = x.mean(-1, keepdim=True)
 s =(x - u ).pow(2 ).mean(-1, keepdim=True)
 x =(x - u)/ torch.sqrt(s + self.e)
 return self.g * x + self.b
class Conv1D(nn.Module):
def __init__(self, nf, rf, nx):
super(Conv1D, self ).__init__()
self.rf = rf
self.nf = nf
if rf == 1:
w = torch.empty(nx, nf)
 nn.init.normal_(w, std=0.02)
 self.w = Parameter(w)
 self.b = Parameter(torch.zeros(nf))
else:
raise NotImplementedError
def forward(self, x):
if self.rf == 1:
size_out = x.size() [:-1] +(self.nf,)
 x = torch.addmm(self.b, x.view(-1, x.size(-1)) , self.w)
 x = x.view(*size_out)
 else:
raise NotImplementedError
return x
class Attention(nn.Module):
def __init__(self, nx, n_ctx, cfg, scale=False):
super(Attention, self ).__init__()
n_state = nx
assert n_state % cfg.n_head == 0
self.register_buffer('b', torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx))
self.n_head = cfg.n_head
self.split_size = n_state
self.scale = scale
self.c_attn = Conv1D(n_state * 3, 1, nx)
 self.c_proj = Conv1D(n_state, 1, nx)
 self.attn_dropout = nn.Dropout(cfg.attn_pdrop)
 self.resid_dropout = nn.Dropout(cfg.resid_pdrop)
 def _attn(self, q, k, v):
w = torch.matmul(q, k)
 if self.scale:
w = w / math.sqrt(v.size(-1))
b = self.b[:, :, :w.size(-2), :w.size(-1)]
w = w * b + -1e9 *(1 - b)
 w = nn.Softmax(dim=-1 )(w)
 w = self.attn_dropout(w)
 return torch.matmul(w, v)
 def merge_heads(self, x):
x = x.permute(0, 2, 1, 3 ).contiguous()
new_x_shape = x.size() [:-2] +(x.size(-2)* x.size(-1),)
 return x.view(*new_x_shape)
 def split_heads(self, x, k=False):
new_x_shape = x.size() [:-1] +(self.n_head, x.size(-1)// self.n_head)
 x = x.view(*new_x_shape)
 if k:
return x.permute(0, 2, 3, 1)
 else:
return x.permute(0, 2, 1, 3)
 def forward(self, x):
x = self.c_attn(x)
 query, key, value = x.split(self.split_size, dim=2)
 query = self.split_heads(query)
 key = self.split_heads(key, k=True)
 value = self.split_heads(value)
 a = self._attn(query, key, value)
 a = self.merge_heads(a)
 a = self.c_proj(a)
 a = self.resid_dropout(a)
 return a
class MLP(nn.Module):
def __init__(self, n_state, cfg):
super(MLP, self ).__init__()
nx = cfg.n_embd
self.c_fc = Conv1D(n_state, 1, nx)
 self.c_proj = Conv1D(nx, 1, n_state)
 self.act = ACT_FNS[cfg.ACT_FNS]
self.dropout = nn.Dropout(cfg.resid_pdrop)
 def forward(self, x):
h = self.act(self.c_fc(x))
h2 = self.c_proj(h)
 return self.dropout(h2)
 class Block(nn.Module):
def __init__(self, n_ctx, cfg, scale=False):
super(Block, self ).__init__()
nx = cfg.n_embd
self.attn = Attention(nx, n_ctx, cfg, scale)
 self.ln_1 = LayerNorm(nx)
 self.mlp = MLP(4 * nx, cfg)
 self.ln_2 = LayerNorm(nx)
 def forward(self, x):
a = self.attn(x)
 n = self.ln_1(x + a)
 m = self.mlp(n)
 h = self.ln_2(n + m)
 return h
class PositionalEncoding(nn.Module):
"Implement the PE function."
def __init__(self, d_model, max_len=5000):
super(PositionalEncoding, self ).__init__()
pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *
-(math.log(10000)/ d_model))
pe[:, 0::2] = torch.sin(torch.as_tensor(position.numpy() * div_term.unsqueeze(0 ).numpy()))
pe[:, 1::2] = torch.cos(torch.as_tensor(position.numpy() * div_term.unsqueeze(0 ).numpy()))
pe = pe.unsqueeze(0)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + Variable(self.pe[:, :x.size(1)],
requires_grad=False)
 return x
class TransformerModel(nn.Module):

 def __init__(self, cfg):
super(TransformerModel, self ).__init__()
self.vocab = cfg.vocab
self.n_ctx = cfg.n_ctx
self.embed = nn.Embedding(self.vocab+1, cfg.n_embd)
 self.PE = PositionalEncoding(cfg.n_embd,self.n_ctx)
 nn.init.normal_(self.embed.weight, std=0.02)
 if cfg.pre_word_Embedding:
self.embed.weight.data.copy_(torch.tensor(cfg.embedding_matrix))
self.embed.weight.requires_grad = True
self.drop = nn.Dropout(cfg.embd_pdrop)
 block = Block(self.n_ctx, cfg, scale=True)
 self.h = nn.ModuleList([copy.deepcopy(block)for _ in range(cfg.n_layer)])
 self.fc = nn.Linear(in_features=cfg.n_embd, out_features=cfg.C, bias=True)
 def forward(self, x):
x = self.embed(x)
 x = self.PE(x)
 h = self.drop(x)
 for block in self.h:
h = block(h)
 logit = self.fc(h[:,-1,:])
 return logit<define_search_space>
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
target = torch.tensor([[0,1,1]], dtype=torch.float32)
 output = torch.tensor([[0.1,0.7,0.8]], dtype=torch.float32)
 criterion = torch.nn.BCEWithLogitsLoss()
criterion(output, target )<compute_test_metric>
out_unscaled = make_outs(test_group, preds )
m=nn.Sigmoid()
m(output )<compute_test_metric>
test['scalar_coupling_constant'] = out_unscaled
test['scalar_coupling_constant'] = test['scalar_coupling_constant']*scale_norm + scale_mid
test[['id','scalar_coupling_constant']].to_csv('submission.csv', index=False )