Kaggle competition: Predicting Molecular Properties
y, label_encoder = prepare_labels(train_df['Id'])
 NCLASSES = len(y[0] )<categorify>
!pip install tensorflow-gpu==2.0a0
trn_trnsfms = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomAffine(degrees=30),
transforms.ToTensor() ,
transforms.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225] 
)
 ])
 tst_trnsfms = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.ToTensor() ,
transforms.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225] 
)
 ] )<categorify>
print(tf.__version__ )
class WhaleDataLoader(Dataset):
def __init__(self, image_folder, process='train', bbox=None, df=None, transform=None, y=None):
self.image_folder = image_folder
self.imgs_list = [img for img in os.listdir(image_folder)]
self.process = process
self.transform = transform
self.y = y
if self.process == 'train':
self.df = df.values
self.bbox = bbox.values
def __len__(self):
return len(self.imgs_list)
 def __getitem__(self, idx):
if self.process == 'train':
img_name = os.path.join(self.image_folder, self.df[idx][0])
 label = self.y[idx]
elif self.process == 'test':
img_name = os.path.join(self.image_folder, self.imgs_list[idx])
 label = np.zeros(( NCLASSES,))
img = Image.open(img_name ).convert('RGB')
 if self.process == 'train':
area =(self.bbox[idx][2], self.bbox[idx][3], self.bbox[idx][4], self.bbox[idx][5])
 elif self.process == 'test':
area =(self.bbox[idx][1], self.bbox[idx][2], self.bbox[idx][3], self.bbox[idx][4])
 img = img.crop(area)
 img = self.transform(img)
 if self.process == 'train':
return img, label
elif self.process == 'test':
return img, label, self.imgs_list[idx]<load_pretrained>
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
train_dataloader = WhaleDataLoader(image_folder = TRN_IMGS_DIR, process='train', bbox=bbox_train, df=train_df, transform=trn_trnsfms, y=y)
 test_dataloader = WhaleDataLoader(image_folder = TST_IMGS_DIR, process='test', bbox=bbox_test,transform=tst_trnsfms )<load_pretrained>
tf.random.set_seed(42)
 datadir = ".. /input/"
batch_size = 10
num_workers = 4
train_loader = DataLoader(train_dataloader, batch_size=batch_size, num_workers=num_workers, pin_memory=True, shuffle=True)
 test_loader = DataLoader(test_dataloader, batch_size=batch_size, num_workers=num_workers, pin_memory=True )<define_search_space>
class Message_Passer_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Message_Passer_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )
pretrained_settings = {
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def se_resnext101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext101_pretrain(model, optimizer, pretrained='checkpoint.pth'):
checkpoint = torch.load(pretrained)
 start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
 optimizer.load_state_dict(checkpoint['optimizer'])
 return model, optimizer, start_epoch
def save_checkpoint(state, is_best, fpath='checkpoint.pth'):
torch.save(state, fpath)
 if is_best:
torch.save(state, 'best_model.pth' )<find_best_params>
class Message_Agg(tf.keras.layers.Layer):
def __init__(self):
super(Message_Agg, self ).__init__()
def call(self, messages):
return tf.math.reduce_sum(messages, 2 )
model = se_resnext101()<choose_model_class>
class Update_Func_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Update_Func_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, old_state, agg_messages):
concat = self.concat_layer([old_state, agg_messages])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation)
model.last_linear = nn.Linear(model.last_linear.in_features, NCLASSES )<choose_model_class>
class Adj_Updater_1(tf.keras.layers.Layer):
def __init__(self, intermediate_dim, state_dim):
super(Adj_Updater_1, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=state_dim, activation = tf.nn.relu)
 def call(self, node_i, node_j, edge_ij):
concat = self.concat_layer([node_i, node_j, edge_ij])
 activation = self.hidden_layer_1(concat)
 return self.output_layer(activation )
model = model.cuda()
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model.parameters() , lr=0.0005 )<find_best_params>
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_3 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1)
 def call(self, edges):
activation_1 = self.hidden_layer_1(edges)
 activation_2 = self.hidden_layer_2(activation_1)
 activation_3 = self.hidden_layer_3(activation_2)
 return self.output_layer(activation_3)
model, optimizer, start_epoch = se_resnext101_pretrain(model, optimizer, best_model_path )<choose_model_class>
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer, self ).__init__(self)
 self.state_dim = state_dim
self.message_passers_1 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_1 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_passers_2 = Message_Passer_1(intermediate_dim = mp_int_dim, state_dim = state_dim)
 self.update_functions_2 = Update_Func_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.batch_norm_n_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_1 = tf.keras.layers.BatchNormalization()
self.batch_norm_e_2 = tf.keras.layers.BatchNormalization()
self.batch_norm_n_2 = tf.keras.layers.BatchNormalization()
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
messages_1 = self.message_passers_1(state_i_1, state_j_1, new_edges_1)
 masked_1 = tf.math.multiply(messages_1, mask)
 masked_1 = tf.reshape(masked_1, [tf.shape(messages_1)[0], tf.shape(nodes_1)[1], tf.shape(nodes_1)[1], tf.shape(messages_1)[2]])
 agg_m_1 = self.message_aggs(masked_1)
 state_1 = self.update_functions_1(nodes_1, agg_m_1)
 messages_2 = self.message_passers_2(state_i_2, state_j_2, new_edges_2)
 masked_2 = tf.math.multiply(messages_2, mask)
 masked_2 = tf.reshape(masked_2, [tf.shape(messages_2)[0], tf.shape(nodes_2)[1], tf.shape(nodes_2)[1], tf.shape(messages_2)[2]])
 agg_m_2 = self.message_aggs(masked_2)
 state_2 = self.update_functions_2(nodes_2, agg_m_2)
 nodes_out_1 = self.batch_norm_n_1(state_1)
 edges_out_1 = self.batch_norm_e_1(new_edges_1)
 nodes_out_2 = self.batch_norm_n_2(state_2)
 edges_out_2 = self.batch_norm_e_2(new_edges_2)
 return nodes_out_1, edges_out_1, nodes_out_2, edges_out_2
scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1 )<find_best_params>
class MP_Layer_edge_only(tf.keras.layers.Layer):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim):
super(MP_Layer_edge_only, self ).__init__(self)
 self.adj_updaters_1 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.adj_updaters_2 = Adj_Updater_1(intermediate_dim = up_int_dim, state_dim = state_dim)
 self.message_aggs = Message_Agg()
self.state_dim = state_dim
def call(self, nodes_branch_1, edges_branch_1, nodes_branch_2, edges_branch_2, mask):
nodes_1 = nodes_branch_1
edges_1 = edges_branch_1
n_nodes = tf.shape(nodes_1)[1]
node_dim = tf.shape(nodes_1)[2]
state_i_1 = tf.tile(nodes_1, [1, n_nodes, 1])
 state_j_1 = tf.reshape(tf.tile(nodes_1, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_1 = self.adj_updaters_1(state_i_1, state_j_1, edges_1)
 new_edges_1 = tf.math.multiply(new_edges_1, mask)
 nodes_2 = nodes_branch_2
edges_2 = edges_branch_2
n_nodes = tf.shape(nodes_2)[1]
node_dim = tf.shape(nodes_2)[2]
state_i_2 = tf.tile(nodes_2, [1, n_nodes, 1])
 state_j_2 = tf.reshape(tf.tile(nodes_2, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 new_edges_2 = self.adj_updaters_2(state_i_2, state_j_2, edges_2)
 new_edges_2 = tf.math.multiply(new_edges_2, mask)
 new_edges_1 = tf.math.multiply(new_edges_1, tf.math.sigmoid(new_edges_2))
return new_edges_1, new_edges_2
n_epochs = 13
mean_losss = 99
is_best = False
for epoch in range(1, n_epochs+1):
train_loss = []
for batch_i,(data, target)in tqdm(enumerate(train_loader), total = len(train_loader)) :
data, target = data.cuda() , target.cuda()
optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target.float())
 train_loss.append(loss.item())
 loss.backward()
optimizer.step()
scheduler.step()
if mean_losss > np.mean(train_loss):
is_best = True
mean_losss = np.mean(train_loss)
 else:
is_best = False
save_checkpoint({'epoch': epoch,
'state_dict': model.state_dict() ,
'optimizer' : optimizer.state_dict() ,
}, is_best)
 print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}' )<save_to_csv>
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, mp_int_dim, up_int_dim, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.MP = [MP_Layer(mp_int_dim, up_int_dim, out_int_dim, state_dim)for _ in range(T)]
self.MP_edge = MP_Layer_edge_only(mp_int_dim, up_int_dim, out_int_dim, state_dim)
 self.embed_node = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.embed_edge = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.edge_regressor = Edge_Regressor(mp_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
edges_0 = edges
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = tf.cast(nodes, dtype=tf.float32)
 edges = tf.cast(edges, dtype=tf.float32)
 mask = tf.cast(mask, dtype=tf.float32)
 nodes_1 = self.embed_node(nodes)
 edges_1 = self.embed_edge(edges)
 nodes_2 = nodes_1
edges_2 = edges_1
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
for i, mp in enumerate(self.MP):
index = i + 1
if index%2 == 0:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 nodes_1 = nodes_1 - nodes_1_
edges_1 = edges_1 - edges_1_
nodes_2 = nodes_2 - nodes_2_
edges_2 = edges_2 - edges_2_
nodes_1_ = nodes_1
edges_1_ = edges_1
nodes_2_ = nodes_2
edges_2_ = edges_2
else:
nodes_1, edges_1, nodes_2, edges_2 = mp(nodes_1, edges_1, nodes_2, edges_2, mask)
 edges_1, edges_2 = self.MP_edge(nodes_1, edges_1, nodes_2, edges_2, mask)
 con_edges = self.edge_regressor(edges_1)
 return con_edges
model.eval()
for(data, target, name)in tqdm(test_loader):
data = data.cuda()
output = model(data)
 output = output.cpu().detach().numpy()
for i,(e, n)in enumerate(list(zip(output, name))):
sub.loc[sub['Image'] == n, 'Id'] = ' '.join(label_encoder.inverse_transform(e.argsort() [-5:][::-1]))
sub.to_csv('submission.csv', index=False )<define_variables>
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error
arch = resnet50
num_workers = 4
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
LABELS = '.. /input/humpback-whale-identification/train.csv'
SAMPLE_SUB = '.. /input/humpback-whale-identification/sample_submission.csv'
BBOX = '.. /input/generating-whale-bounding-boxes/bounding_boxes.csv'<feature_engineering>
learning_rate = 0.001
def warmup(epoch):
initial_lrate = learning_rate
if epoch == 0:
lrate = 0.00001
if epoch == 1:
lrate = 0.0001
if epoch > 1:
lrate = 0.001
if epoch > 20:
lrate = 0.0001
if epoch > 25:
lrate = 0.00001
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(warmup)
 opt = tf.optimizers.Adam(learning_rate=learning_rate)
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]<count_unique_values>
mpnn = MPNN(mp_int_dim = 512, up_int_dim = 1024, out_int_dim = 512, state_dim = 256, T = 3)
 mpnn.compile(opt, log_mae)
train_df['image_name'] = train_df.index
tr_n = train_df['image_name'].values
val_n = train_df['image_name']
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<create_dataframe>
batch_size = 64
epochs = 30
def get_data(sz, batch_size):

 aug_tfms = []
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md<define_variables>
def train() :
nodes_train = np.load(datadir + "internalgraphdata/nodes_train.npz")['arr_0']
in_edges_train = np.load(datadir + "internalgraphdata/in_edges_train.npz")['arr_0']
out_edges_train = np.load(datadir + "internalgraphdata/out_edges_train.npz")['arr_0']
out_labels = out_edges_train.reshape(-1,out_edges_train.shape[1]*out_edges_train.shape[2],1)
 in_edges_train = in_edges_train.reshape(-1,in_edges_train.shape[1]*in_edges_train.shape[2],in_edges_train.shape[3])
 train_size = int(len(out_labels)*0.8)
 mpnn.call({'adj_input' : in_edges_train[:10], 'nod_input': nodes_train[:10]})
 history = mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))
preds = mpnn.predict({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]}, verbose = 0)
 return preds, train_size, history
image_size = 384
batch_size = 32
md = get_data(image_size, batch_size )<define_variables>
%%time
preds, train_size, history = train()
best_th = 0.38<load_pretrained>
mpnn.save_weights("model.h5" )
preds_t = np.load(".. /input/humpbackdata/resnet101.npy")+ np.load(".. /input/humpbackdata/resnet50.npy")
 preds_t /= 2<save_to_csv>
with open('/trainHistoryDict.pkl', 'wb')as file_pi:
pickle.dump(history.history, file_pi )
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(md.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('submission.csv', header=True, index=False)
 df.head()<install_modules>
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 train_mol_names = train['molecule_name'].unique()
val = train[train.molecule_name.isin(train_mol_names[train_size:])]
val_group = val.groupby('molecule_name' )
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1<define_variables>
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
MODEL_NAME = 'Resnet101'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = resnet101
num_workers = 4<feature_engineering>
max_size = 29
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(val_group, preds )
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]<feature_engineering>
val['pred_scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]*scale_norm + scale_mid
val.loc[val.type == coup, 'pred_scalar_coupling_constant'] = val['pred_scalar_coupling_constant'].loc[val.type == coup]
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<prepare_x_and_y>
for coup in coups_to_isolate:
log_mae = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'][val.type == coup])
 print(coup,"\t", log_mae)
 total = group_mean_log_mae(val['scalar_coupling_constant'], val['pred_scalar_coupling_constant'], val['type'])
 print("")
 print("Total:","\t", total )
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels )<categorify>
nodes_test = np.load(datadir + "internalgraphdata/nodes_test.npz")['arr_0']
in_edges_test = np.load(datadir + "internalgraphdata/in_edges_test.npz")['arr_0']
in_edges_test = in_edges_test.reshape(-1,in_edges_test.shape[1]*in_edges_test.shape[2],in_edges_test.shape[3] )
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md<load_pretrained>
preds = mpnn.predict({'adj_input' : in_edges_test, 'nod_input': nodes_test}, verbose=1 )
image_size = 299
batch_size = 32
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam<init_hyperparams>
np.save("preds_kernel.npy" , preds )
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]<train_model>
test_group = test.groupby('molecule_name' )
base_lr = 5e-4
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn_bn_unfreeze()
learn.fit(lrs, n_cycle=4, cycle_len=1, cycle_mult=2 )<train_model>
preds = preds.reshape(( -1,max_size, max_size))
out_unscaled = make_outs(test_group, preds )
base_lr = 1e-4
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs, n_cycle=6, cycle_len=1 )<prepare_output>
test['scalar_coupling_constant'] = out_unscaled
coups_to_isolate = ['1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN']
for i, coup in enumerate(coups_to_isolate):
scale_min = train['scalar_coupling_constant'].loc[train.type == coup].min()
scale_max = train['scalar_coupling_constant'].loc[train.type == coup].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid
test.loc[test.type == coup, 'scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]*scale_norm + scale_mid
test.loc[test.type == coup, 'pred_scalar_coupling_constant'] = test['scalar_coupling_constant'].loc[test.type == coup]
<save_to_csv><EOS> 
test[['id','scalar_coupling_constant']].to_csv('submission.csv', index=False )