Kaggle competition: Predicting Molecular Properties
import pandas as pd
import subprocess
import torch
import torch.optim as optim
from torch import nn
from torch.utils.data import Dataset
from torch.utils.data.sampler import SubsetRandomSampler
import torchvision
from torchvision import transforms
import os
import random
from glob import glob
import cv2
import numpy as np
from tqdm.notebook import tqdm<define_variables>
!pip install tensorflow-gpu==2.0a0
EPOCH = 20
BATCH_SIZE = 16
PATIENCE = 5
DATA_PATH = '.. /input/state-farm-distracted-driver-detection'<categorify>
print(tf.__version__ )
transform = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] )<load_from_csv>
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
classes = [f'c{i}' for i in range(10)]
seed = 2020
validation_split = 0.2
driver_list = pd.read_csv(f'{DATA_PATH}/driver_imgs_list.csv')
 drivers = np.unique(driver_list['subject'].values)
 split = int(np.floor(validation_split * len(drivers)))
 np.random.seed(seed)
 trn_idx, val_idx = drivers[split:], drivers[:split]<set_options>
tf.random.set_seed(42)
 datadir = ".. /input/"
! ls -ahl driver_split<load_pretrained>
class Message_Passer_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Message_Passer_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )
train_dataset = torchvision.datasets.ImageFolder(f'./{split_dir}/train',
transform=transform)
 train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2)
 valid_dataset = torchvision.datasets.ImageFolder(f'./{split_dir}/valid',
transform=transform)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=BATCH_SIZE,
num_workers=2 )<set_options>
class Message_Agg(tf.keras.layers.Layer):
def __init__(self):
super(Message_Agg, self ).__init__()
def call(self, messages):
return tf.math.reduce_sum(messages, 2 )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )<choose_model_class>
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
model_conv = torchvision.models.resnet50(pretrained=True )<choose_model_class>
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )
num_ftrs = model_conv.fc.in_features
model_conv.fc = nn.Sequential(
 nn.Linear(num_ftrs, num_ftrs),
nn.ReLU() ,
nn.Dropout(0.5),
nn.Linear(num_ftrs, num_ftrs),
nn.ReLU() ,
nn.Dropout(0.5),
nn.Linear(num_ftrs, len(classes)))
 print(f'
model_conv = model_conv.to(device )<choose_model_class>
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_conv.parameters() , lr=0.0001, weight_decay=1e-6, momentum=0.9)
 softmax = nn.Softmax(dim=1)
 best_valid_score = 999
patience = 0<train_model>
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers_1 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_1 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_passers_2 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_2 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_2 = tf.keras.layers.BatchNormalization()
self.batch_norm_n_2 = tf.keras.layers.BatchNormalization()
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
messages_1 = self.message_passers_1(state_i_1, state_j_1, new_edges_1)
 masked_1 = tf.math.multiply(messages_1, mask)
 masked_1 = tf.reshape(masked_1, [tf.shape(messages_1)[0], tf.shape(nodes_1)[1], tf.shape(nodes_1)[1], tf.shape(messages_1)[2]])
 agg_m_1 = self.message_aggs(masked_1)
 state_1 = self.update_functions_1(nodes_1, agg_m_1)
 messages_2 = self.message_passers_2(state_i_2, state_j_2, new_edges_2)
 masked_2 = tf.math.multiply(messages_2, mask)
 masked_2 = tf.reshape(masked_2, [tf.shape(messages_2)[0], tf.shape(nodes_2)[1], tf.shape(nodes_2)[1], tf.shape(messages_2)[2]])
 agg_m_2 = self.message_aggs(masked_2)
 state_2 = self.update_functions_2(nodes_2, agg_m_2)
 state_1 = tf.math.multiply(state_1, tf.math.sigmoid(state_2))
nodes_out_1 = self.batch_norm_n_1(state_1)
 edges_out_1 = self.batch_norm_e_1(new_edges_1)
 nodes_out_2 = self.batch_norm_n_2(state_2)
 edges_out_2 = self.batch_norm_e_2(new_edges_2)
 return nodes_out_1, edges_out_1, nodes_out_2, edges_out_2
def train(model_conv, train_loader, optimizer, criterion, trn_cnt):
running_loss = 0.
 running_acc = 0.
 pbar = tqdm(total=trn_cnt)
 cnt = 0
for i, data in enumerate(train_loader, 0):
inputs, labels = data
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
model_conv.train()
outputs = model_conv(inputs)
 probs = softmax(outputs)
 _, preds = probs.max(axis=1)
 running_acc += sum(labels == preds)/(1.* BATCH_SIZE)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
cnt += 1
pbar.update(BATCH_SIZE)
 pbar.close()
return running_loss / cnt, running_acc / cnt<init_hyperparams>
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
return new_edges_1, new_edges_2
def evaluate(model_conv, valid_loader, criterion):
with torch.no_grad() :
model_conv.eval()
valid_loss = 0.0
valid_acc = 0.0
cnt = 0
pbar = tqdm(total=val_cnt)
 for data in valid_loader:
inputs, labels = data
inputs = inputs.to(device)
 labels = labels.to(device)
 outputs = model_conv(inputs)
 probs = softmax(outputs)
 _, preds = probs.max(axis=1)
 valid_acc += sum(labels == preds)/(1.* BATCH_SIZE)
 labels.require_grad = False
loss = criterion(outputs, labels)
 valid_loss += loss
cnt += 1
pbar.update(BATCH_SIZE)
 pbar.close()
return valid_loss / cnt, valid_acc / cnt<train_model>
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = tf.cast(nodes, dtype=tf.float32)
 edges = tf.cast(edges, dtype=tf.float32)
 mask = tf.cast(mask, dtype=tf.float32)
 nodes_1 = self.embed_node(nodes)
 edges_1 = self.embed_edge(edges)
 nodes_2 = nodes_1
edges_2 = edges_1
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 nodes_1 = nodes_1 - nodes_1_
edges_1 = edges_1 - edges_1_
nodes_2 = nodes_2 - nodes_2_
edges_2 = edges_2 - edges_2_
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
else:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 edges_1, edges_2 = self.MP_edge(nodes_1, edges_1, nodes_2, edges_2, mask)
 con_edges = self.edge_regressor(edges_1)
 return con_edges
for epoch in range(EPOCH):
print(f'
trn_loss, trn_acc = train(model_conv, train_loader, optimizer, criterion, trn_cnt)
 print(f'
valid_loss, valid_acc = evaluate(model_conv, valid_loader, criterion)
 print(f'
if valid_loss < best_valid_score:
best_valid_score = valid_loss
print(f'
torch.save(model_conv, './model.baseline.driver_split')
 patience = 0
patience += 1
if patience == PATIENCE:
break
print('Finished Training' )<define_variables>
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error
def mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.abs(tf.subtract(nums, preds)))
 return reconstruction_error
TEST_SIZE = 79726
BATCH_SIZE = 128
test_ids = [os.path.basename(fl)for fl in glob(f'{DATA_PATH}/imgs/test/img_*.jpg')]
test_ids.sort()
transform = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 test_dataset = torchvision.datasets.ImageFolder(f'{DATA_PATH}/imgs',
transform=transform)
 test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=BATCH_SIZE,
num_workers=2)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 model_conv = torch.load('model.baseline.driver_split')
 print(f'
model_conv = model_conv.to(device)
 softmax = nn.Softmax(dim=1)
 pbar = tqdm(total=TEST_SIZE)
 end_flag = False
with open('submission.csv', 'w')as out:
out.write('img,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9
')
 for i, data in enumerate(test_loader, 0):
inputs, labels = data
inputs = inputs.to(device)
 labels = labels.to(device)
 with torch.no_grad() :
model_conv.eval()
outputs = model_conv(inputs)
 probs = softmax(outputs)
 for j, prob in enumerate(probs):
if BATCH_SIZE * i + j >= TEST_SIZE:
end_flag = True
break
test_id = test_ids[i * BATCH_SIZE + j]
prob = ','.join([str(round(val, 3)) for val in prob.cpu().detach().numpy() ])
 out.write(f'{test_id},{prob}
')
 pbar.update(BATCH_SIZE)
 if end_flag:
break
pbar.close()
print('Finished Eval' )<set_options>
learning_rate = 0.001
def step_decay(epoch):
initial_lrate = learning_rate
drop = 0.1
epochs_drop = 20.0
lrate = initial_lrate * np.power(drop,
np.floor(( epoch)/epochs_drop))
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(step_decay)
 early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 15, restore_best_weights=True)
 opt = tf.optimizers.Adam(learning_rate=learning_rate )
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline<load_from_csv>
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 3)
 mpnn.compile(opt, log_mae, metrics = [mae, log_mae])
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape<prepare_x_and_y>
batch_size = 64
epochs = 30
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<feature_engineering>
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 history = mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size, history
for df in [train, test]:
df['date'] = pd.to_datetime(df['date'])
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day<categorify>
%%time
preds, train_size, history = train()
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )<define_variables>
mpnn.save_weights("model.h5" )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
with open('/trainHistoryDict.pkl', 'wb')as file_pi:
pickle.dump(history.history, file_pi )
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<prepare_x_and_y>
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=500 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
max_size = 29
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(val_group, preds )
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
val['pred_scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]*scale_norm + scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
for coup in coups_to_isolate:
log_mae = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'][val.type == coup])
 print(coup,"\t", log_mae)
 total = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'])
 print("")
 print("Total:","\t", total )
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
nodes_test = np.load(datadir + "internalgraphdata/nodes_test.npz")['arr_0']
in_edges_test = np.load(datadir + "internalgraphdata/in_edges_test.npz")['arr_0']
in_edges_test = in_edges_test.reshape(-1,in_edges_test.shape[1]*in_edges_test.shape[2],in_edges_test.shape[3] )
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
preds = mpnn.predict({'adj_input' : in_edges_test, 'nod_input': nodes_test}, verbose=1 )
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
np.save("preds_kernel.npy" , preds )
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<prepare_x_and_y>
test_group = test.groupby('molecule_name' )
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=500 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(test_group, preds )
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )<set_options>
test['scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
test.loc[test.type == coup, 'scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]*scale_norm + scale_mid
test.loc[test.type == coup, 'pred_scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]
<feature_engineering><EOS> 
test[['id','scalar_coupling_constant']].to_csv('submission.csv', index=False )