Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
nltk.download('stopwords', quiet=True)
 stopwords = stopwords.words('english')
 sns.set(style="white", font_scale=1.2)
 plt.rcParams["figure.figsize"] = [10,8]
pd.set_option.display_max_columns = 0
pd.set_option.display_max_rows = 0
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
<choose_model_class>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )<choose_model_class>
null_counts = pd.DataFrame({"Num_Null": train.isnull().sum() })
 null_counts["Pct_Null"] = null_counts["Num_Null"] / train.count() * 100
null_counts
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
<choose_model_class>
len(train["keyword"].value_counts() )
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers_1 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_1 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_passers_2 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_2 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_2 = tf.keras.layers.BatchNormalization()
self.batch_norm_n_2 = tf.keras.layers.BatchNormalization()
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
messages_1 = self.message_passers_1(state_i_1, state_j_1, new_edges_1)
 masked_1 = tf.math.multiply(messages_1, mask)
 masked_1 = tf.reshape(masked_1, [tf.shape(messages_1)[0], tf.shape(nodes_1)[1], tf.shape(nodes_1)[1], tf.shape(messages_1)[2]])
 agg_m_1 = self.message_aggs(masked_1)
 state_1 = self.update_functions_1(nodes_1, agg_m_1)
 messages_2 = self.message_passers_2(state_i_2, state_j_2, new_edges_2)
 masked_2 = tf.math.multiply(messages_2, mask)
 masked_2 = tf.reshape(masked_2, [tf.shape(messages_2)[0], tf.shape(nodes_2)[1], tf.shape(nodes_2)[1], tf.shape(messages_2)[2]])
 agg_m_2 = self.message_aggs(masked_2)
 state_2 = self.update_functions_2(nodes_2, agg_m_2)
 state_1 = tf.math.multiply(state_1, tf.math.sigmoid(state_2))
nodes_out_1 = self.batch_norm_n_1(state_1)
 edges_out_1 = self.batch_norm_e_1(new_edges_1)
 nodes_out_2 = self.batch_norm_n_2(state_2)
 edges_out_2 = self.batch_norm_e_2(new_edges_2)
 return nodes_out_1, edges_out_1, nodes_out_2, edges_out_2<choose_model_class>
def keyword_disaster_probabilities(x):
tweets_w_keyword = np.sum(train["keyword"].fillna("" ).str.contains(x))
tweets_w_keyword_disaster = np.sum(train["keyword"].fillna("" ).str.contains(x)& train["target"] == 1)
 return tweets_w_keyword_disaster / tweets_w_keyword
keywords_vc["Disaster_Probability"] = keywords_vc.index.map(keyword_disaster_probabilities)
 keywords_vc.head()
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
return new_edges_1, new_edges_2<choose_model_class>
keywords_vc.sort_values(by="Disaster_Probability", ascending=False ).head(10 )
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = tf.cast(nodes, dtype=tf.float32)
 edges = tf.cast(edges, dtype=tf.float32)
 mask = tf.cast(mask, dtype=tf.float32)
 nodes_1 = self.embed_node(nodes)
 edges_1 = self.embed_edge(edges)
 nodes_2 = nodes_1
edges_2 = edges_1
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 nodes_1 = nodes_1 - nodes_1_
edges_1 = edges_1 - edges_1_
nodes_2 = nodes_2 - nodes_2_
edges_2 = edges_2 - edges_2_
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
else:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 edges_1, edges_2 = self.MP_edge(nodes_1, edges_1, nodes_2, edges_2, mask)
 con_edges = self.edge_regressor(edges_1)
 return con_edges<compute_test_metric>
len(train["location"].value_counts() )
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error
def mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.abs(tf.subtract(nums, preds)))
 return reconstruction_error<choose_model_class>
def create_corpus(target):
corpus = []
for w in train.loc[train["target"] == target]["text"].str.split() :
for i in w:
corpus.append(i)
 return corpus
def create_corpus_dict(target):
corpus = create_corpus(target)
 stop_dict = defaultdict(int)
 for word in corpus:
if word in stopwords:
stop_dict[word] += 1
return sorted(stop_dict.items() , key=lambda x:x[1], reverse=True )
learning_rate = 0.001
def step_decay(epoch):
initial_lrate = learning_rate
drop = 0.1
epochs_drop = 20.0
lrate = initial_lrate * np.power(drop,
np.floor(( epoch)/epochs_drop))
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(step_decay)
 early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 15, restore_best_weights=True)
 opt = tf.optimizers.Adam(learning_rate=learning_rate )<choose_model_class>
corpus_disaster, corpus_non_disaster = create_corpus(1), create_corpus(0)
 counter_disaster, counter_non_disaster = Counter(corpus_disaster), Counter(corpus_non_disaster)
 x_disaster, y_disaster, x_non_disaster, y_non_disaster = [], [], [], []
counter = 0
for word, count in counter_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_disaster.append(word)
 y_disaster.append(count)
 counter = 0
for word, count in counter_non_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_non_disaster.append(word)
 y_non_disaster.append(count)
 fig, ax = plt.subplots(1,2, figsize=(20,8))
sns.barplot(x=y_disaster, y=x_disaster, orient='h', palette="Reds_d", ax=ax[0])
 sns.barplot(x=y_non_disaster, y=x_non_disaster, orient='h', palette="Blues_d", ax=ax[1])
 ax[0].set_title("Top 15 Non-Stopwords - Disaster Tweets")
 ax[0].set_xlabel("Word Frequency")
 ax[1].set_title("Top 15 Non-Stopwords - Non-Disaster Tweets")
 ax[1].set_xlabel("Word Frequency")
 plt.tight_layout()
plt.show()
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 3)
 mpnn.compile(opt, log_mae, metrics = [mae, log_mae])
<define_variables>
def bigrams(target):
corpus = train[train["target"] == target]["text"]
count_vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = count_vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in count_vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq
batch_size = 64
epochs = 30 
<train_model>
def remove_pattern(input_txt, pattern):
r = re.findall(pattern, input_txt)
 for i in r:
input_txt = re.sub(i, '', input_txt)
 return input_txt
train['tweet'] = np.vectorize(remove_pattern )(train['text'], "
test['tweet'] = np.vectorize(remove_pattern )(test['text'], "
train.head()
train['tweet'] = train['tweet'].str.replace("[^a-zA-Z
test['tweet'] = test['tweet'].str.replace("[^a-zA-Z
train.head()
train['tweet'] = train['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
test['tweet'] = test['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
train.head()
train['tweet'] = train['tweet'].str.lower()
test['tweet'] = test['tweet'].str.lower()
set(stopwords.words('english'))
stops = set(stopwords.words('english'))
train['tokenized_sents'] = train.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 test['tokenized_sents'] = test.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 def remove_stops(row):
my_list = row['tokenized_sents']
meaningful_words = [w for w in my_list if not w in stops]
return(meaningful_words)
 train['clean_tweet'] = train.apply(remove_stops, axis=1)
 test['clean_tweet'] = test.apply(remove_stops, axis=1)
 train.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 test.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 def rejoin_words(row):
my_list = row['clean_tweet']
joined_words =(" ".join(my_list))
return joined_words
train['clean_tweet'] = train.apply(rejoin_words, axis=1)
 test['clean_tweet'] = test.apply(rejoin_words, axis=1)
 train.head()
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 history = mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size, history<train_model>
warnings.filterwarnings("ignore")
 tqdm.pandas()
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
 random_state = 29
%%time
preds, train_size, history = train()<load_pretrained>
!pip install GPUtil
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()
mpnn.save_weights("model.h5" )<load_pretrained>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sub= pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
with open('/trainHistoryDict.pkl', 'wb')as file_pi:
pickle.dump(history.history, file_pi )<load_from_csv>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )<compute_test_metric>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'URL',text)
 def remove_HTML(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_not_ASCII(text):
text = ''.join([word for word in text if word in string.printable])
 return text
def word_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def replace_abbrev(text):
string = ""
for word in text.split() :
string += word_abbrev(word)+ " "
return string
def remove_mention(text):
at=re.compile(r'@\S+')
 return at.sub(r'USER',text)
 def remove_number(text):
num = re.compile(r'[-+]?[.\d]*[\d]+[:,.\d]*')
 return num.sub(r'NUMBER', text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'EMOJI', text)
 def transcription_sad(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[(\\/]')
 return smiley.sub(r'SADFACE', text)
 def transcription_smile(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[)dDp]')
 return smiley.sub(r'SMILE', text)
 def transcription_heart(text):
heart = re.compile(r'<3')
 return heart.sub(r'HEART', text)
 def remove_elongated_words(text):
rep = re.compile(r'\b(\S*? )([a-z])\2{2,}\b')
 return rep.sub(r'\1\2 ELONG', text)
 def remove_repeat_punct(text):
rep = re.compile(r'([!?.]){2,}')
 return rep.sub(r'\1 REPEAT', text)
 def remove_all_punct(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_punct(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
def remove_stopwords(text):
text = ' '.join([word for word in text.split() if word not in(stopwords)])
 return text
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()<normalization>
def clean_tweet(text):
text = remove_URL(text)
 text = remove_HTML(text)
 text = remove_not_ASCII(text)
 text = text.lower()
text = replace_abbrev(text)
 text = remove_mention(text)
 text = remove_number(text)
 text = remove_emoji(text)
 text = transcription_sad(text)
 text = transcription_smile(text)
 text = transcription_heart(text)
 text = remove_elongated_words(text)
 text = remove_repeat_punct(text)
 return text
max_size = 29
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(val_group, preds )<feature_engineering>
train["clean_text"] = train["text"].apply(clean_tweet)
 test["clean_text"] = test["text"].apply(clean_tweet)
 train["clean_tokens"] = train["clean_text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["clean_text"].apply(lambda x: word_tokenize(x))
val['pred_scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]*scale_norm + scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]<compute_train_metric>
skip_gram_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2,sg=1)
 skip_gram_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 cbow_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2)
 cbow_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 train["clean_tokens"] = train["text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["text"].apply(lambda x: word_tokenize(x))
def get_word_embeddings(token_list,vector,k=150):
if len(token_list)< 1:
return np.zeros(k)
 else:
vectorized = [vector[word] if word in vector else np.random.rand(k)for word in token_list]
sum = np.sum(vectorized,axis=0)
 return sum/len(vectorized )
for coup in coups_to_isolate:
log_mae = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'][val.type == coup])
 print(coup,"\t", log_mae)
 total = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'])
 print("")
 print("Total:","\t", total )<load_pretrained>
max_features=5000
count_vectorizer = CountVectorizer(max_features=max_features)
 sparce_matrix_train=count_vectorizer.fit_transform(train['clean_text'])
 sparce_matrix_test=count_vectorizer.fit_transform(train['clean_text'])
 def count_vector(data):
count_vectorizer = CountVectorizer()
vect = count_vectorizer.fit_transform(data)
 return vect, count_vectorizer
def tfidf_vector(data):
tfidf_vectorizer = TfidfVectorizer()
vect = tfidf_vectorizer.fit_transform(data)
 return vect, tfidf_vectorizer
def skip_gram_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, skip_gram_model))
return list(embeddings)
 def cbow_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, cbow_model))
return list(embeddings)
 X_train_count, count_vectorizer = count_vector(train['clean_text'])
 X_train_tfidf, tfidf_vectorizer = tfidf_vector(train['clean_text'])
 X_train_skip_gram = skip_gram_vectorizer(train['clean_tokens'],skip_gram_model)
 X_train_cbow = cbow_vectorizer(train['clean_tokens'],cbow_model)
 X_test_count = count_vectorizer.transform(test['clean_text'])
 X_test_tfidf = tfidf_vectorizer.transform(test['clean_text'])
 X_test_skip_gram = skip_gram_vectorizer(test['clean_tokens'],skip_gram_model)
 X_test_cbow = cbow_vectorizer(test['clean_tokens'],cbow_model)
 X_train_skip_gram = np.array(X_train_skip_gram)
 X_train_cbow = np.array(X_train_cbow)
 X_test_skip_gram = np.array(X_test_skip_gram)
 X_test_cbow = np.array(X_test_cbow)
 with open('count_vectorizer.pickle', 'wb')as handle:
pickle.dump(count_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('tfidf_vectorizer.pickle', 'wb')as handle:
pickle.dump(tfidf_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('skip_gram_model.pickle', 'wb')as handle:
pickle.dump(skip_gram_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
nodes_test = np.load(datadir + "internalgraphdata/nodes_test.npz")['arr_0']
in_edges_test = np.load(datadir + "internalgraphdata/in_edges_test.npz")['arr_0']
in_edges_test = in_edges_test.reshape(-1,in_edges_test.shape[1]*in_edges_test.shape[2],in_edges_test.shape[3] )<predict_on_test>
metrics = pd.DataFrame(columns=['model' ,'vectoriser', 'f1 score', 'train accuracy','test accuracy'] )
preds = mpnn.predict({'adj_input' : in_edges_test, 'nod_input': nodes_test}, verbose=1 )<save_model>
models=[
XGBClassifier(max_depth=6, n_estimators=1000),
LogisticRegression(random_state=random_state),
SVC(random_state=random_state),
MultinomialNB() ,
DecisionTreeClassifier(random_state = random_state),
KNeighborsClassifier() ,
RandomForestClassifier(random_state=random_state),
]
np.save("preds_kernel.npy" , preds )<groupby>
for model in models:
y = train.target
x = X_train_count
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test,'Count vector')
 x = X_train_tfidf
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'Tfidf vector')
 x = X_train_skip_gram
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'skip gram vector')
 x = X_train_cbow
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'cbow vector' )
test_group = test.groupby('molecule_name' )<normalization>
metrics = metrics.sort_values('f1 score',ascending=False )
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(test_group, preds )<feature_engineering>
free_gpu_cache()
test['scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
test.loc[test.type == coup, 'scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]*scale_norm + scale_mid
test.loc[test.type == coup, 'pred_scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup] 
<save_to_csv>
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM,GRU, Dropout, Activation, Input, Flatten, Bidirectional, Conv1D, MaxPooling1D
from tensorflow.keras import layers
from keras import optimizers
from functools import reduce
test[['id','scalar_coupling_constant']].to_csv('submission.csv', index=False )<install_modules>
def train_lstm(x_train,x_test,y_train,y_test,vectorizer_name,vocab_size,input_length):
epochs = 1
verbose = 1
batch_size = 32
embed_dim = 32
optimizer = optimizers.Adam(lr=0.002)
 model = Sequential()
model.add(Embedding(vocab_size, embed_dim,input_length = input_length))
model.add(Dropout(0.2))
model.add(LSTM(32, dropout=0.2, recurrent_dropout=0.4))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss = 'binary_crossentropy', optimizer=optimizer, metrics = ['accuracy'])
 print(model.summary())
 model.fit(x_train, y_train, epochs = epochs, verbose = verbose, batch_size=batch_size, validation_data=(x_test, y_test))
model_name = "LSTM using "+ vectorizer_name
model.save(model_name)
 y_pred = model.predict(x_test)
 f1score = f1_score(y_test,y_pred.round() ,average='weighted')
 train_accuracy = round(accuracy_score(y_train,reduce(list.__add__, model.predict(x_train ).round().tolist())) *100)
 test_accuracy = round(accuracy_score(y_test,y_pred.round())*100)
 global metrics
metrics = metrics.append({
'model': model_name,
'f1 score': f1score,
'train accuracy': train_accuracy,
'test accuracy': test_accuracy,
'vectoriser': vectorizer_name,
},
ignore_index=True 
)
 print(model_name)
 print(classification_report(y_test,y_pred.round()))
print('Accuracy of classifier on training set:{}%'.format(train_accuracy))
print('Accuracy of classifier on test set:{}%'.format(test_accuracy))
!pip install tensorflow-gpu==2.0a0<import_modules>
y = train['target'].values
x_train, x_test, y_train, y_test = train_test_split(X_train_skip_gram,y, test_size = 0.3)
 train_lstm(x_train,x_test,y_train,y_test, 'skip gram vector',5329,150)
print(tf.__version__ )<set_options>
%reset -f
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
<define_variables>
!pip install GPUtil
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()
free_gpu_cache()
tf.random.set_seed(42)
 datadir = ".. /input/"<choose_model_class>
import re
import torch
from transformers import ElectraTokenizer, ElectraForSequenceClassification,AdamW
import torch
from sklearn.metrics import classification_report
import random
import time
import datetime
import numpy as np
import pandas as pd
from transformers import get_linear_schedule_with_warmup
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from sklearn.model_selection import train_test_split
class Message_Passer_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Message_Passer_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )<normalization>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
class Message_Agg(tf.keras.layers.Layer):
def __init__(self):
super(Message_Agg, self ).__init__()
def call(self, messages):
return tf.math.reduce_sum(messages, 2 )<choose_model_class>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 df_train= train
df_test= test
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
<choose_model_class>
def preprocess(text):
text=text.lower()
text = re.sub(r'https?:\/\/.*[\r
]*', '', text)
 text = re.sub(r'http?:\/\/.*[\r
]*', '', text)
 text=text.replace(r'&amp;?',r'and')
 text=text.replace(r'&lt;',r'<')
 text=text.replace(r'&gt;',r'>')
 text = re.sub(r"(?:\@)\w+", '', text)
 text=text.encode("ascii",errors="ignore" ).decode()
text=re.sub(r'[:"
text=re.sub(r'[!]+','!',text)
 text=re.sub(r'[?]+','?',text)
 text=re.sub(r'[.]+','.',text)
 text=re.sub(r"'","",text)
 text=re.sub(r"\(","",text)
 text=re.sub(r"\)","",text)
 text=" ".join(text.split())
 return text
df_train['text'] = df_train['text'].apply(preprocess)
 df_test['text'] = df_test['text'].apply(preprocess)
 df_train=df_train[df_train["text"]!='']
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )<choose_model_class>
df_train=df_train[["text","target"]]
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
<choose_model_class>
texts = df_train.text.values
labels = df_train.target.values
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n = tf.keras.layers.BatchNormalization()
self.batch_norm_e = tf.keras.layers.BatchNormalization()
def call(self, nodes, edges, mask):
nodes_0 = nodes
edges_0 = edges
n_nodes = tf.shape(nodes_0)[1]
node_dim = tf.shape(nodes_0)[2]
state_i = tf.tile(nodes_0, [1, n_nodes, 1])
 state_j = tf.reshape(tf.tile(nodes_0, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges = self.adj_updaters(state_i, state_j, edges_0)
 new_edges = tf.math.multiply(new_edges, mask)
 messages = self.message_passers(state_i, state_j, new_edges)
 masked = tf.math.multiply(messages, mask)
 masked = tf.reshape(masked, [tf.shape(messages)[0], tf.shape(nodes_0)[1], tf.shape(nodes_0)[1], tf.shape(messages)[2]])
 agg_m = self.message_aggs(masked)
 state_1 = self.update_functions(nodes_0, agg_m)
 nodes_out = self.batch_norm_n(state_1)
 edges_out = self.batch_norm_e(new_edges)
 return nodes_out, edges_out<choose_model_class>
torch.cuda.empty_cache()
tokenizer = ElectraTokenizer.from_pretrained('google/electra-base-discriminator')
 model = ElectraForSequenceClassification.from_pretrained('google/electra-base-discriminator',num_labels=2)
 model.cuda()
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes, edges, mask):
nodes_0 = nodes
edges_0 = edges
n_nodes = tf.shape(nodes_0)[1]
node_dim = tf.shape(nodes_0)[2]
state_i = tf.tile(nodes_0, [1, n_nodes, 1])
 state_j = tf.reshape(tf.tile(nodes_0, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges = self.adj_updaters(state_i, state_j, edges_0)
 new_edges = tf.math.multiply(new_edges, mask)
 edges_out = new_edges
return edges_out<choose_model_class>
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = self.embed_node(nodes)
 edges = self.embed_edge(edges)
 nodes_ = nodes
edges_ = edges
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes, edges = mp(nodes, edges, mask)
 nodes = nodes - nodes_
edges = edges - edges_
nodes_ = nodes
edges_ = edges
else:
nodes, edges = mp(nodes, edges, mask)
 edges = self.MP_edge(nodes, edges, mask)
 con_edges = self.edge_regressor(edges)
 return con_edges<compute_test_metric>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error
def mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.abs(tf.subtract(nums, preds)))
 return reconstruction_error<choose_model_class>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 7)
 mpnn.compile(opt, log_mae, metrics = [mae, log_mae])
<define_variables>
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )
batch_size = 64
epochs = 30 
<train_model>
optimizer = AdamW(model.parameters() ,
lr = 6e-6,
eps = 1e-8 
)
 epochs = 5
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 history = mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate, early_stop], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size, history<find_best_params>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
preds, train_size, history = train()<load_from_csv>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )<compute_test_metric>
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
preds=[]
true=[]
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
preds.append(logits)
 true.append(label_ids)
 tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print("Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print("Validation took: {:}".format(format_time(time.time() - t0)) )
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()<normalization>
flat_predictions = [item for sublist in preds for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
flat_true_labels = [item for sublist in true for item in sublist]
max_size = 29
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(val_group, preds )<feature_engineering>
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
val['pred_scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]*scale_norm + scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]<compute_train_metric>
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
for coup in coups_to_isolate:
log_mae = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'][val.type == coup])
 print(coup,"\t", log_mae)
 total = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'])
 print("")
 print("Total:","\t", total )<load_pretrained>
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':flat_predictions} )
<predict_on_test><EOS> 
df_leak = pd.read_csv('/kaggle/input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding ='ISO-8859-1')[['choose_one', 'text']]
df_leak['target'] =(df_leak['choose_one'] == 'Relevant' ).astype(np.int8)
 df_leak['id'] = df_leak.index.astype(np.int16)
 df_leak.drop(columns=['choose_one', 'text'], inplace=True)
 df_test = df_test.merge(df_leak, on=['id'], how='left')
 print('Leaked Data Set Shape = {}'.format(df_leak.shape))
print('Leaked Data Set Memory Usage = {:.2f} MB'.format(df_leak.memory_usage().sum() / 1024**2))
print(df_test)
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = df_test['target'].values
submission.to_csv('submission.csv', index=False)
 print(submission )