Kaggle competition: Bike Sharing Demand
<SOS> metric: RMSLE Kaggle data source: bike-sharing-demand<choose_model_class>
%matplotlib inline
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
<choose_model_class>
train['datetime'] = pd.to_datetime(train['datetime'],errors='coerce')
 train['date'] = train['datetime'].apply(lambda x: x.date())
 train['year'] = train['datetime'].apply(lambda x: x.year)
 train['month'] = train['datetime'].apply(lambda x: x.month)
 train['weekday'] = train['datetime'].apply(lambda x: x.weekday())
 train['hour'] = train['datetime'].apply(lambda x: x.hour ).astype('int' )
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers_1 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_1 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_passers_2 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_2 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_2 = tf.keras.layers.BatchNormalization()
self.batch_norm_n_2 = tf.keras.layers.BatchNormalization()
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
messages_1 = self.message_passers_1(state_i_1, state_j_1, new_edges_1)
 masked_1 = tf.math.multiply(messages_1, mask)
 masked_1 = tf.reshape(masked_1, [tf.shape(messages_1)[0], tf.shape(nodes_1)[1], tf.shape(nodes_1)[1], tf.shape(messages_1)[2]])
 agg_m_1 = self.message_aggs(masked_1)
 state_1 = self.update_functions_1(nodes_1, agg_m_1)
 messages_2 = self.message_passers_2(state_i_2, state_j_2, new_edges_2)
 masked_2 = tf.math.multiply(messages_2, mask)
 masked_2 = tf.reshape(masked_2, [tf.shape(messages_2)[0], tf.shape(nodes_2)[1], tf.shape(nodes_2)[1], tf.shape(messages_2)[2]])
 agg_m_2 = self.message_aggs(masked_2)
 state_2 = self.update_functions_2(nodes_2, agg_m_2)
 nodes_out_1 = self.batch_norm_n_1(state_1)
 edges_out_1 = self.batch_norm_e_1(new_edges_1)
 nodes_out_2 = self.batch_norm_n_2(state_2)
 edges_out_2 = self.batch_norm_e_2(new_edges_2)
 return nodes_out_1, edges_out_1, nodes_out_2, edges_out_2<import_modules>
dummies_month = pd.get_dummies(train['month'], prefix= 'month')
 dummies_season = pd.get_dummies(train['season'], prefix= 'season')
 dummies_weather = pd.get_dummies(train['weather'], prefix= 'weather')
 dummies_year = pd.get_dummies(train['year'], prefix= 'year')
 data=pd.concat([train,dummies_month,dummies_season,dummies_weather,dummies_year],axis=1)
 yLabels=data['count']
dropFeatures = ['casual' , 'count' , 'datetime' ,
'registered' , 'date' ,'season',
'weather','month','year']
dataTrain = data.drop(dropFeatures,axis=1 )
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
return new_edges_1, new_edges_2<choose_model_class>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_log_error
from math import sqrt
import statsmodels.api as sm
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = tf.cast(nodes, dtype=tf.float32)
 edges = tf.cast(edges, dtype=tf.float32)
 mask = tf.cast(mask, dtype=tf.float32)
 nodes_1 = self.embed_node(nodes)
 edges_1 = self.embed_edge(edges)
 nodes_2 = nodes_1
edges_2 = edges_1
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 nodes_1 = nodes_1 - nodes_1_
edges_1 = edges_1 - edges_1_
nodes_2 = nodes_2 - nodes_2_
edges_2 = edges_2 - edges_2_
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
else:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 edges_1, edges_2 = self.MP_edge(nodes_1, edges_1, nodes_2, edges_2, mask)
 con_edges = self.edge_regressor(edges_1)
 return con_edges<compute_test_metric>
X_train_ca, X_valid_ca, y_train_ca, y_valid_ca = train_test_split(dataTrain,
train['casual'], test_size=0.3, random_state=42)
 X_train_re, X_valid_re, y_train_re, y_valid_re = train_test_split(dataTrain,
train['registered'], test_size=0.3, random_state=42)
 X_train, X_valid, y_train, y_valid = train_test_split(dataTrain,
train['count'], test_size=0.3, random_state=42 )
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error<choose_model_class>
rfModel_ca = RandomForestRegressor(n_estimators=1000 ,
oob_score=True, random_state = 42)
 rfModel_ca.fit(X_train_ca , y_train_ca)
 rfModel_re = RandomForestRegressor(n_estimators=1000 ,
oob_score=True, random_state = 42)
 rfModel_re.fit(X_train_re , y_train_re)
 rfModel = RandomForestRegressor(n_estimators=1000 ,
oob_score=True, random_state = 42)
 rfModel.fit(X_train , y_train )
learning_rate = 0.001
def warmup(epoch):
initial_lrate = learning_rate
if epoch == 0:
lrate = 0.00001
if epoch == 1:
lrate = 0.0001
if epoch > 1:
lrate = 0.001
if epoch > 20:
lrate = 0.0001
if epoch > 25:
lrate = 0.00001
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(warmup)
 opt = tf.optimizers.Adam(learning_rate=learning_rate)
<choose_model_class>
preds_train_ca = rfModel_ca.predict(X = X_train_ca)
 rmsle_casual_train = sqrt(mean_squared_log_error(y_train_ca, preds_train_ca))
preds_valid_ca = rfModel_ca.predict(X = X_valid_ca)
 rmsle_casual_valid = sqrt(mean_squared_log_error(y_valid_ca, preds_valid_ca))
print('Casual train rmsle : %.5f, valid rmsle : %.5f' %(rmsle_casual_train,
rmsle_casual_valid))
preds_train_re = rfModel_re.predict(X = X_train_re)
 rmsle_registered_train = sqrt(mean_squared_log_error(y_train_re, preds_train_re))
preds_valid_re = rfModel_re.predict(X = X_valid_re)
 rmsle_registered_valid = sqrt(mean_squared_log_error(y_valid_re, preds_valid_re))
print('Registered train rmsle : %.5f,valid rmsle : %.5f' %(rmsle_registered_train,
rmsle_registered_valid))
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 3)
 mpnn.compile(opt, log_mae)
<define_variables>
preds_train = rfModel.predict(X = X_train)
 rmsle_count_train = sqrt(mean_squared_log_error(y_train, preds_train))
preds_valid = rfModel.predict(X = X_valid)
 rmsle_count_valid = sqrt(mean_squared_log_error(y_valid, preds_valid))
print('Count train rmsle : %.5f, valid rmsle : %.5f' %(rmsle_count_train,
rmsle_count_valid))
batch_size = 64
epochs = 30 
<train_model>
preds_train_merge_count = preds_train_re + preds_train_ca
preds_valid_merge_count = preds_valid_re + preds_valid_ca
rmsle_merge_train = sqrt(mean_squared_log_error(y_train,preds_train_merge_count))
rmsle_merge_valid = sqrt(mean_squared_log_error(y_valid,preds_valid_merge_count))
print('(merge)Count train rmsle : %.5f, valid rmsle : %.5f'%(rmsle_merge_train,
rmsle_merge_valid))
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 history = mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size, history<train_model>
rf_dict={'casual':rfModel_ca.feature_importances_.round(3),
'registered':rfModel_re.feature_importances_.round(3),
'count':rfModel.feature_importances_.round(3)}
pd.DataFrame(rf_dict,X_train_ca.columns )
%%time
preds, train_size, history = train()<load_pretrained>
test['datetime'] = pd.to_datetime(test['datetime'],errors='coerce')
 test['date'] = test['datetime'].apply(lambda x: x.date())
 test['year'] = test['datetime'].apply(lambda x: x.year)
 test['month'] = test['datetime'].apply(lambda x: x.month)
 test['weekday'] = test['datetime'].apply(lambda x: x.weekday())
 test['hour'] = test['datetime'].apply(lambda x: x.hour ).astype('int' )
mpnn.save_weights("model.h5" )<load_pretrained>
dummies_month = pd.get_dummies(test['month'], prefix= 'month')
 dummies_season = pd.get_dummies(test['season'], prefix= 'season')
 dummies_weather = pd.get_dummies(test['weather'], prefix= 'weather')
 dummies_year = pd.get_dummies(test['year'], prefix= 'year')
 data_test=pd.concat([test,dummies_month,dummies_season,dummies_weather,dummies_year],axis=1)
 datetimecol = test['datetime']
yLabels=data['count']
dropFeatures = ['datetime' , 'date' ,'season',
'weather','month','year']
dataTest = data_test.drop(dropFeatures,axis=1 )
with open('/trainHistoryDict.pkl', 'wb')as file_pi:
pickle.dump(history.history, file_pi )<load_from_csv>
predsTest= rfModel_re.predict(X = dataTest)+ rfModel_ca.predict(X = dataTest)
 submission=pd.DataFrame({'datetime':datetimecol , 'count':[max(0,x)for x in predsTest]} )
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )<compute_test_metric>
submission.to_csv('sampleSubmission.csv',index=False )