Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<feature_engineering>
!pip install tensorflow-gpu==2.0a0
path1=".. /input/ga-customer-revenue-prediction/"
path2='.. /input/gstore-prepared-dataset/'
data_files=['.. /input/gstore-prepared-dataset/prepared_train/prepared_train',
'.. /input/gstore-prepared-dataset/prepared_test/prepared_test']
df_files=pd.DataFrame(data_files,columns=['File_Name'])
 df_files['Size_in_MB']=df_files.File_Name.apply(lambda x:round(os.stat(x ).st_size/(1024*1024),2))
df_files<feature_engineering>
print(tf.__version__ )
def feature_summary(df_fa):
print('DataFrame shape')
 print('rows:',df_fa.shape[0])
 print('cols:',df_fa.shape[1])
 col_list=['Null','Unique_Count','Data_type','Max/Min','Mean','Std','Skewness','Sample_values']
df=pd.DataFrame(index=df_fa.columns,columns=col_list)
 df['Null']=list([len(df_fa[col][df_fa[col].isnull() ])for i,col in enumerate(df_fa.columns)])
 df['Unique_Count']=list([len(df_fa[col].unique())for i,col in enumerate(df_fa.columns)])
 df['Data_type']=list([df_fa[col].dtype for i,col in enumerate(df_fa.columns)])
 for i,col in enumerate(df_fa.columns):
if 'float' in str(df_fa[col].dtype)or 'int' in str(df_fa[col].dtype):
df.at[col,'Max/Min']=str(round(df_fa[col].max() ,2)) +'/'+str(round(df_fa[col].min() ,2))
df.at[col,'Mean']=df_fa[col].mean()
df.at[col,'Std']=df_fa[col].std()
df.at[col,'Skewness']=df_fa[col].skew()
df.at[col,'Sample_values']=list(df_fa[col].unique())
 return(df.fillna('-'))
def get_val(x,col):
try:
y=x[col]
except:
y=np.nan
return(y)
 def rsme(y,pred):
return(mean_squared_error(y,pred)**0.5 )<load_from_csv>
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
%%time
print('reading train dataset...')
 df_train=pd.read_csv(data_files[0],dtype={'fullVisitorId':str})
 print('reading test dataset...')
 df_test=pd.read_csv(data_files[1],dtype={'fullVisitorId':str})
 print('data reading complete' )<create_dataframe>
tf.random.set_seed(42)
 datadir = ".. /input/"
train_cpy=df_train.copy()<feature_engineering>
class Message_Passer_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Message_Passer_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )
df_train['revenue_status']=df_train.totals_transactionRevenue.apply(lambda x: 0 if x==0 else 1 )<sort_values>
class Message_Agg(tf.keras.layers.Layer):
def __init__(self):
super(Message_Agg, self ).__init__()
def call(self, messages):
return tf.math.reduce_sum(messages, 2 )
df_browser=df_train[['device_browser','totals_transactionRevenue','revenue_status']].groupby(df_train.device_browser ).aggregate({'totals_transactionRevenue':['mean'],
'revenue_status':['count']} ).reset_index()
df_browser.columns=['device_browser','totals_transactionRevenue_mean','revenue_status_count']
df=df_browser.sort_values(by='totals_transactionRevenue_mean',ascending=False)[df_browser.totals_transactionRevenue_mean>0]
formatter = {'totals_transactionRevenue_mean':'{:4.2f}'}
display(df.style.format(formatter))
plt.subplots(figsize=(20,5))
plt.subplot(1,2,1)
 plt.title('REVENUE MEAN BY BROWSER',color='b',fontsize=12)
 plt.xlabel('Browsers',color='b',fontsize=12)
 plt.ylabel('Mean Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.totals_transactionRevenue_mean,color='grey')
 plt.xticks(range(len(df)) ,df.device_browser,rotation=90,fontsize=12)
 plt.yticks(fontsize=12)
 plt.subplot(1,2,2)
 plt.title('NUMBER OF TRANSACTIONS WITH REVENUE BY BROWSER',color='b',fontsize=12)
 plt.xlabel('Browsers',color='b',fontsize=12)
 plt.ylabel('Count of Transactions with Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.revenue_status_count,color='orange')
 plt.xticks(range(len(df)) ,df.device_browser,rotation=90,fontsize=12)
 plt.yticks(fontsize=12)
 plt.show()<groupby>
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
df_OS=df_train[['device_operatingSystem','totals_transactionRevenue','revenue_status']].groupby(df_train.device_operatingSystem ).aggregate({'totals_transactionRevenue':['mean'],
'revenue_status':['count']} ).reset_index()
df_OS.columns=['device_operatingSystem','totals_transactionRevenue_mean','revenue_status_count']
df=df_OS.sort_values(by='totals_transactionRevenue_mean',ascending=False)[df_OS.totals_transactionRevenue_mean>0]
display(df.style.format(formatter))
plt.subplots(figsize=(20,5))
plt.subplot(1,2,1)
 plt.title('REVENUE MEAN BY OPERATING SYSTEM',color='b',fontsize=12)
 plt.xlabel('Operating Systems',color='b',fontsize=12)
 plt.ylabel('Mean Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.totals_transactionRevenue_mean,color='grey')
 plt.xticks(range(len(df)) ,df.device_operatingSystem,rotation=90,fontsize=12)
 plt.yticks(fontsize=12)
 plt.subplot(1,2,2)
 plt.title('NUMBER OF TRANSACTIONS WITH REVENUE BY OPERATING SYSTEM',color='b',fontsize=12)
 plt.xlabel('Operating Systems',color='b',fontsize=12)
 plt.ylabel('Count of Transactions with Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.revenue_status_count,color='orange')
 plt.yticks(fontsize=12)
 plt.xticks(range(len(df)) ,df.device_operatingSystem,rotation=90,fontsize=12)
 plt.show()<groupby>
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )
df_isM=df_train[['device_isMobile','totals_transactionRevenue','revenue_status']].groupby(df_train.device_isMobile ).aggregate({'totals_transactionRevenue':['mean'],
'revenue_status':['count']} ).reset_index()
df_isM.columns=['device_isMobile','totals_transactionRevenue_mean','revenue_status_count']
df=df_isM.sort_values(by='totals_transactionRevenue_mean',ascending=False)[df_isM.totals_transactionRevenue_mean>0]
display(df.style.format(formatter))
plt.subplots(figsize=(20,5))
plt.subplot(1,2,1)
 plt.title('REVENUE MEAN BY MOBILE',color='b',fontsize=12)
 plt.xlabel('Mobile',color='b',fontsize=12)
 plt.ylabel('Mean Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.totals_transactionRevenue_mean,color='grey')
 plt.xticks(range(len(df)) ,df.device_isMobile,rotation=90,fontsize=12)
 plt.yticks(fontsize=12)
 plt.subplot(1,2,2)
 plt.title('NUMBER OF TRANSACTIONS WITH REVENUE BY MOBILE',color='b',fontsize=12)
 plt.xlabel('Mobile',color='b',fontsize=12)
 plt.ylabel('Count of Transactions with Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.revenue_status_count,color='orange')
 plt.yticks(fontsize=12)
 plt.xticks(range(len(df)) ,df.device_isMobile,rotation=90,fontsize=12)
 plt.show()<compute_test_metric>
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
revenue_transaction_visualization(df_train,'geoNetwork_country','COUNTRY' )<compute_train_metric>
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n = tf.keras.layers.BatchNormalization()
self.batch_norm_e = tf.keras.layers.BatchNormalization()
def call(self, nodes, edges, mask):
nodes_0 = nodes
edges_0 = edges
n_nodes = tf.shape(nodes_0)[1]
node_dim = tf.shape(nodes_0)[2]
state_i = tf.tile(nodes_0, [1, n_nodes, 1])
 state_j = tf.reshape(tf.tile(nodes_0, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges = self.adj_updaters(state_i, state_j, edges_0)
 new_edges = tf.math.multiply(new_edges, mask)
 messages = self.message_passers(state_i, state_j, new_edges)
 masked = tf.math.multiply(messages, mask)
 masked = tf.reshape(masked, [tf.shape(messages)[0], tf.shape(nodes_0)[1], tf.shape(nodes_0)[1], tf.shape(messages)[2]])
 agg_m = self.message_aggs(masked)
 state_1 = self.update_functions(nodes_0, agg_m)
 nodes_out = self.batch_norm_n(state_1)
 edges_out = self.batch_norm_e(new_edges)
 return nodes_out, edges_out
revenue_transaction_visualization(df_train,'geoNetwork_city','CITY' )<create_dataframe>
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes, edges, mask):
nodes_0 = nodes
edges_0 = edges
n_nodes = tf.shape(nodes_0)[1]
node_dim = tf.shape(nodes_0)[2]
state_i = tf.tile(nodes_0, [1, n_nodes, 1])
 state_j = tf.reshape(tf.tile(nodes_0, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges = self.adj_updaters(state_i, state_j, edges_0)
 new_edges = tf.math.multiply(new_edges, mask)
 edges_out = new_edges
return edges_out
revenue_transaction_visualization(df_train,'geoNetwork_networkDomain','NETWORKDOMAIN' )<create_dataframe>
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = self.embed_node(nodes)
 edges = self.embed_edge(edges)
 nodes_ = nodes
edges_ = edges
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes, edges = mp(nodes, edges, mask)
 nodes = nodes - nodes_
edges = edges - edges_
nodes_ = nodes
edges_ = edges
else:
nodes, edges = mp(nodes, edges, mask)
 edges = self.MP_edge(nodes, edges, mask)
 con_edges = self.edge_regressor(edges)
 return con_edges
revenue_transaction_visualization(df_train,'geoNetwork_subContinent','SUBCONTINENT' )<create_dataframe>
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error
def mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.abs(tf.subtract(nums, preds)))
 return reconstruction_error
revenue_transaction_visualization(df_train,'geoNetwork_continent','CONTINENT' )<create_dataframe>
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 7)
 mpnn.compile(opt, log_mae, metrics = [mae, log_mae])
revenue_transaction_visualization(df_train,'geoNetwork_metro','METRO' )<create_dataframe>
batch_size = 64
epochs = 30
revenue_transaction_visualization(df_train,'trafficSource_adContent','AdCONTENT' )<count_unique_values>
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 history = mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate, early_stop], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size, history
print('Train data: Unique AdContent value count')
 display(df_train['trafficSource_adContent'].groupby(df_train.trafficSource_adContent ).count())
 print('Test data: Unique AdContent value count')
 display(df_test['trafficSource_adContent'].groupby(df_test.trafficSource_adContent ).count())
<groupby>
preds, train_size, history = train()
df_train['trafficSource_isTrueDirect'].replace({np.nan:'False'},inplace=True)
 df_isTD=df_train[['trafficSource_isTrueDirect','totals_transactionRevenue','revenue_status']].groupby(df_train.trafficSource_isTrueDirect ).aggregate({'totals_transactionRevenue':['mean'],
'revenue_status':['count']} ).reset_index()
df_isTD.columns=['trafficSource_isTrueDirect','totals_transactionRevenue_mean','revenue_status_count']
df=df_isTD.sort_values(by='totals_transactionRevenue_mean',ascending=False)[df_isTD.totals_transactionRevenue_mean>0]
display(df.style.format(formatter))
plt.subplots(figsize=(20,5))
plt.subplot(1,2,1)
 plt.title('REVENUE MEAN BY isTrueDirect',color='b',fontsize=12)
 plt.xlabel('isTrueDirect',color='b',fontsize=12)
 plt.ylabel('Mean Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.totals_transactionRevenue_mean,color='grey')
 plt.xticks(range(len(df)) ,df.trafficSource_isTrueDirect,rotation=90,fontsize=12)
 plt.yticks(fontsize=12)
 plt.subplot(1,2,2)
 plt.title('NUMBER OF TRANSACTIONS WITH REVENUE BY isTrueDirect',color='b',fontsize=12)
 plt.xlabel('isTrueDirect',color='b',fontsize=12)
 plt.ylabel('Count of Transactions with Revenue',color='b',fontsize=12)
 plt.bar(range(len(df)) ,df.revenue_status_count,color='orange')
 plt.yticks(fontsize=12)
 plt.xticks(range(len(df)) ,df.trafficSource_isTrueDirect,rotation=90,fontsize=12)
 plt.show()<compute_test_metric>
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )
revenue_transaction_visualization(df_train,'trafficSource_medium','MEDIUM' )<create_dataframe>
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
revenue_transaction_visualization(df_train,'trafficSource_referralPath','referralPath' )<feature_engineering>
max_size = 29
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(val_group, preds )
df_combi['date'] = pd.to_datetime(df_combi['visitStartTime'], unit='s')
 df_combi['day_of_week'] = df_combi['date'].dt.dayofweek
df_combi['hour'] = df_combi['date'].dt.hour
df_combi['day'] = df_combi['date'].dt.day
df_combi['month'] = df_combi['date'].dt.month
df_combi['revenue_status']=df_combi.totals_transactionRevenue.apply(lambda x: 0 if x==0 else 1 )<feature_engineering>
val['pred_scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]*scale_norm + scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]
%%time
for col in df_combi.columns:
if(( df_combi[col].dtype=='object')&(col!='fullVisitorId')) :
df_combi[col]=df_combi[col].apply(lambda x:str(x ).lower())
 df_combi.replace('nan',np.nan,inplace=True )<create_dataframe>
for coup in coups_to_isolate:
log_mae = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'][val.type == coup])
 print(coup,"\t", log_mae)
 total = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'])
 print("")
 print("Total:","\t", total )
%%time
df_combi.drop(['device_isMobile'],axis=1,inplace=True)
 cat_col=['channelGrouping','device_deviceCategory','tsadclick_slot','tsadclick_adNetworkType','tsadclick_isVideoAd','trafficSource_medium',
'geoNetwork_continent']
for col in cat_col:
df_combi[col]=df_combi[col].apply(lambda x: str(x ).replace(" ","_"))
dummy=pd.DataFrame()
for col in cat_col:
if col.find('_')!=-1:
col_name=col.split('_')[1]
else:
col_name=col
dummy=pd.concat([dummy,pd.get_dummies(df_combi[col],prefix=col_name)],axis=1)
 print('Newly created dummy cols:',len(dummy.columns))
df_combi=pd.concat([df_combi,dummy],axis=1)
 df_combi.drop(cat_col,axis=1,inplace=True )<split>
nodes_test = np.load(datadir + "internalgraphdata/nodes_test.npz")['arr_0']
in_edges_test = np.load(datadir + "internalgraphdata/in_edges_test.npz")['arr_0']
in_edges_test = in_edges_test.reshape(-1,in_edges_test.shape[1]*in_edges_test.shape[2],in_edges_test.shape[3] )
train=df_combi[:len(train_cpy)]
test=df_combi[len(train_cpy):]<feature_engineering>
preds = mpnn.predict({'adj_input' : in_edges_test, 'nod_input': nodes_test}, verbose=1 )
%%time
train['geoNetwork_networkDomain'].fillna('unknown.unknown',inplace=True)
 test['geoNetwork_networkDomain'].fillna('unknown.unknown',inplace=True)
 train['geoNetwork_networkDomain']=train.geoNetwork_networkDomain.apply(lambda x: x.replace('.',' '))
test['geoNetwork_networkDomain']=test.geoNetwork_networkDomain.apply(lambda x: x.replace('.',' '))<categorify>
np.save("preds_kernel.npy" , preds )
%%time
Tvect=TfidfVectorizer(ngram_range=(1,2),max_features=20000)
 vect=Tvect.fit(train['geoNetwork_networkDomain'])
 train_vect=vect.transform(train['geoNetwork_networkDomain'])
 test_vect=vect.transform(test['geoNetwork_networkDomain'])
 svd=TruncatedSVD(n_components=10)
 vect_cols=['vect'+str(x)for x in range(1,11)]
df_train_vect=pd.DataFrame(svd.fit_transform(train_vect),columns=vect_cols)
 df_test_vect=pd.DataFrame(svd.fit_transform(test_vect),columns=vect_cols )<drop_column>
test_group = test.groupby('molecule_name' )
X=train.drop(['sessionId','visitId','date','geoNetwork_networkDomain','tsadclick_gclId'],axis=1)
 X_test=test.drop(['sessionId','visitId','date','geoNetwork_networkDomain','tsadclick_gclId'],axis=1 )<concatenate>
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(test_group, preds )
X=pd.concat([X,df_train_vect],axis=1)
 X_test=pd.concat([X_test,df_test_vect],axis=1 )<define_variables>
test['scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
test.loc[test.type == coup, 'scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]*scale_norm + scale_mid
test.loc[test.type == coup, 'pred_scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]
%%time
agg_func={}
agg_col=['fullVisitorId']
for col in [x for x in X.columns if x not in ['fullVisitorId']]:
if col=='totals_transactionRevenue':
agg_func[col]=['sum']
agg_col.append(str(col)+'_sum')
 elif col=='revenue_status':
agg_func[col]=['sum']
agg_col.append(str(col)+'_sum')
 else:
agg_func[col]=['sum','max','min','mean','var','std']
agg_col.append(str(col)+'_sum')
 agg_col.append(str(col)+'_max')
 agg_col.append(str(col)+'_min')
 agg_col.append(str(col)+'_mean')
 agg_col.append(str(col)+'_var')
 agg_col.append(str(col)+'_std')
 X=X.groupby(X.fullVisitorId ).aggregate(agg_func ).reset_index()
X.columns=agg_col
X_test=X_test.groupby(X_test.fullVisitorId ).aggregate(agg_func ).reset_index()
X_test.columns=agg_col<prepare_x_and_y>
test[['id','scalar_coupling_constant']].to_csv('submission.csv', index=False )