Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<count_unique_values>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore" )
train_df.shape, train_df.Id.nunique()<count_unique_values>
file_folder = '.. /input/champs-scalar-coupling'
train = pd.read_csv(f'{file_folder}/train.csv')
 test = pd.read_csv(f'{file_folder}/test.csv')
 sub = pd.read_csv(f'{file_folder}/sample_submission.csv')
 structures = pd.read_csv(f'{file_folder}/structures.csv' )
NUM_CLASSES = train_df.Id.nunique()<normalization>
path = '.. /input/parallelization-of-coulomb-yukawa-interaction'
structures_yukawa = pd.read_csv(f'{path}/structures_yukawa.csv')
 structures = pd.concat([structures, structures_yukawa], axis=1)
 del structures_yukawa
RESIZE_H = 299
RESIZE_W = 299
data_transforms = albumentations.Compose([
albumentations.Resize(RESIZE_H, RESIZE_W),
albumentations.HorizontalFlip() ,
albumentations.OneOf([
albumentations.RandomContrast() ,
albumentations.RandomBrightness() ,
]),
albumentations.ShiftScaleRotate(rotate_limit=10, scale_limit=0.15),
albumentations.JpegCompression(80),
albumentations.HueSaturationValue() ,
AT.ToTensor()
])
 data_transforms_test = albumentations.Compose([
albumentations.Resize(RESIZE_H, RESIZE_W),
AT.ToTensor()
] )<categorify>
y = train['scalar_coupling_constant']
train = train.drop(columns = ['scalar_coupling_constant'] )
def prepare_labels(y):
values = np.array(y)
 label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
 onehot_encoder = OneHotEncoder(sparse=False)
 integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
 onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
 y = onehot_encoded
return y, label_encoder<categorify>
def map_atom_info(df_1, df_2, atom_idx):
df = pd.merge(df_1, df_2, how = 'left',
left_on = ['molecule_name', f'atom_index_{atom_idx}'],
right_on = ['molecule_name', 'atom_index'])
 df = df.drop('atom_index', axis=1)
 return df
def find_dist(df):
df_p_0 = df[['x_0', 'y_0', 'z_0']].values
df_p_1 = df[['x_1', 'y_1', 'z_1']].values
df['dist'] = np.linalg.norm(df_p_0 - df_p_1, axis=1)
 df['dist_inv2'] = 1/df['dist']**2
df['dist_x'] =(df['x_0'] - df['x_1'])** 2
df['dist_y'] =(df['y_0'] - df['y_1'])** 2
df['dist_z'] =(df['z_0'] - df['z_1'])** 2
return df
def find_closest_atom(df):
df_temp = df.loc[:,["molecule_name",
"atom_index_0","atom_index_1",
"dist","x_0","y_0","z_0","x_1","y_1","z_1"]].copy()
df_temp_ = df_temp.copy()
df_temp_ = df_temp_.rename(columns={'atom_index_0': 'atom_index_1',
'atom_index_1': 'atom_index_0',
'x_0': 'x_1',
'y_0': 'y_1',
'z_0': 'z_1',
'x_1': 'x_0',
'y_1': 'y_0',
'z_1': 'z_0'})
 df_temp_all = pd.concat(( df_temp,df_temp_),axis=0)
 df_temp_all["min_distance"]=df_temp_all.groupby(['molecule_name',
'atom_index_0'])['dist'].transform('min')
 df_temp_all["max_distance"]=df_temp_all.groupby(['molecule_name',
'atom_index_0'])['dist'].transform('max')
 df_temp = df_temp_all[df_temp_all["min_distance"]==df_temp_all["dist"]].copy()
df_temp = df_temp.drop(['x_0','y_0','z_0','min_distance'], axis=1)
 df_temp = df_temp.rename(columns={'atom_index_0': 'atom_index',
'atom_index_1': 'atom_index_closest',
'dist': 'distance_closest',
'x_1': 'x_closest',
'y_1': 'y_closest',
'z_1': 'z_closest'})
 df_temp = df_temp.drop_duplicates(subset=['molecule_name', 'atom_index'])
 for atom_idx in [0,1]:
df = map_atom_info(df,df_temp, atom_idx)
 df = df.rename(columns={'atom_index_closest': f'atom_index_closest_{atom_idx}',
'distance_closest': f'distance_closest_{atom_idx}',
'x_closest': f'x_closest_{atom_idx}',
'y_closest': f'y_closest_{atom_idx}',
'z_closest': f'z_closest_{atom_idx}'})
 df_temp= df_temp_all[df_temp_all["max_distance"]==df_temp_all["dist"]].copy()
df_temp = df_temp.drop(['x_0','y_0','z_0','max_distance'], axis=1)
 df_temp= df_temp.rename(columns={'atom_index_0': 'atom_index',
'atom_index_1': 'atom_index_farthest',
'dist': 'distance_farthest',
'x_1': 'x_farthest',
'y_1': 'y_farthest',
'z_1': 'z_farthest'})
 df_temp = df_temp.drop_duplicates(subset=['molecule_name', 'atom_index'])
 for atom_idx in [0,1]:
df = map_atom_info(df,df_temp, atom_idx)
 df = df.rename(columns={'atom_index_farthest': f'atom_index_farthest_{atom_idx}',
'distance_farthest': f'distance_farthest_{atom_idx}',
'x_farthest': f'x_farthest_{atom_idx}',
'y_farthest': f'y_farthest_{atom_idx}',
'z_farthest': f'z_farthest_{atom_idx}'})
 return df
def add_cos_features(df):
df["distance_center0"] = np.sqrt(( df['x_0']-df['c_x'])**2 \
+(df['y_0']-df['c_y'])**2 \
+(df['z_0']-df['c_z'])**2)
 df["distance_center1"] = np.sqrt(( df['x_1']-df['c_x'])**2 \
+(df['y_1']-df['c_y'])**2 \
+(df['z_1']-df['c_z'])**2)
 df['distance_c0'] = np.sqrt(( df['x_0']-df['x_closest_0'])**2 + \ 
(df['y_0']-df['y_closest_0'])**2 + \ 
(df['z_0']-df['z_closest_0'])**2)
 df['distance_c1'] = np.sqrt(( df['x_1']-df['x_closest_1'])**2 + \ 
(df['y_1']-df['y_closest_1'])**2 + \ 
(df['z_1']-df['z_closest_1'])**2)
 df["distance_f0"] = np.sqrt(( df['x_0']-df['x_farthest_0'])**2 + \ 
(df['y_0']-df['y_farthest_0'])**2 + \ 
(df['z_0']-df['z_farthest_0'])**2)
 df["distance_f1"] = np.sqrt(( df['x_1']-df['x_farthest_1'])**2 + \ 
(df['y_1']-df['y_farthest_1'])**2 + \ 
(df['z_1']-df['z_farthest_1'])**2)
 vec_center0_x =(df['x_0']-df['c_x'])/(df["distance_center0"]+1e-10)
 vec_center0_y =(df['y_0']-df['c_y'])/(df["distance_center0"]+1e-10)
 vec_center0_z =(df['z_0']-df['c_z'])/(df["distance_center0"]+1e-10)
 vec_center1_x =(df['x_1']-df['c_x'])/(df["distance_center1"]+1e-10)
 vec_center1_y =(df['y_1']-df['c_y'])/(df["distance_center1"]+1e-10)
 vec_center1_z =(df['z_1']-df['c_z'])/(df["distance_center1"]+1e-10)
 vec_c0_x =(df['x_0']-df['x_closest_0'])/(df["distance_c0"]+1e-10)
 vec_c0_y =(df['y_0']-df['y_closest_0'])/(df["distance_c0"]+1e-10)
 vec_c0_z =(df['z_0']-df['z_closest_0'])/(df["distance_c0"]+1e-10)
 vec_c1_x =(df['x_1']-df['x_closest_1'])/(df["distance_c1"]+1e-10)
 vec_c1_y =(df['y_1']-df['y_closest_1'])/(df["distance_c1"]+1e-10)
 vec_c1_z =(df['z_1']-df['z_closest_1'])/(df["distance_c1"]+1e-10)
 vec_f0_x =(df['x_0']-df['x_farthest_0'])/(df["distance_f0"]+1e-10)
 vec_f0_y =(df['y_0']-df['y_farthest_0'])/(df["distance_f0"]+1e-10)
 vec_f0_z =(df['z_0']-df['z_farthest_0'])/(df["distance_f0"]+1e-10)
 vec_f1_x =(df['x_1']-df['x_farthest_1'])/(df["distance_f1"]+1e-10)
 vec_f1_y =(df['y_1']-df['y_farthest_1'])/(df["distance_f1"]+1e-10)
 vec_f1_z =(df['z_1']-df['z_farthest_1'])/(df["distance_f1"]+1e-10)
 vec_x =(df['x_1']-df['x_0'])/df['dist']
vec_y =(df['y_1']-df['y_0'])/df['dist']
vec_z =(df['z_1']-df['z_0'])/df['dist']
df["cos_c0_c1"] = vec_c0_x*vec_c1_x + vec_c0_y*vec_c1_y + vec_c0_z*vec_c1_z
df["cos_f0_f1"] = vec_f0_x*vec_f1_x + vec_f0_y*vec_f1_y + vec_f0_z*vec_f1_z
df["cos_c0_f0"] = vec_c0_x*vec_f0_x + vec_c0_y*vec_f0_y + vec_c0_z*vec_f0_z
df["cos_c1_f1"] = vec_c1_x*vec_f1_x + vec_c1_y*vec_f1_y + vec_c1_z*vec_f1_z
df["cos_center0_center1"] = vec_center0_x*vec_center1_x \
+ vec_center0_y*vec_center1_y \
+ vec_center0_z*vec_center1_z
df["cos_c0"] = vec_c0_x*vec_x + vec_c0_y*vec_y + vec_c0_z*vec_z
df["cos_c1"] = vec_c1_x*vec_x + vec_c1_y*vec_y + vec_c1_z*vec_z
df["cos_f0"] = vec_f0_x*vec_x + vec_f0_y*vec_y + vec_f0_z*vec_z
df["cos_f1"] = vec_f1_x*vec_x + vec_f1_y*vec_y + vec_f1_z*vec_z
df["cos_center0"] = vec_center0_x*vec_x + vec_center0_y*vec_y + vec_center0_z*vec_z
df["cos_center1"] = vec_center1_x*vec_x + vec_center1_y*vec_y + vec_center1_z*vec_z
return df
def add_dist_features(df):
df[f'molecule_atom_index_0_dist_mean'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('mean')
 df[f'molecule_atom_index_0_dist_mean_diff'] = df[f'molecule_atom_index_0_dist_mean'] - df['dist']
df[f'molecule_atom_index_0_dist_min'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('min')
 df[f'molecule_atom_index_0_dist_min_diff'] = df[f'molecule_atom_index_0_dist_min'] - df['dist']
df[f'molecule_atom_index_0_dist_std'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('std')
 df[f'molecule_atom_index_1_dist_mean'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('mean')
 df[f'molecule_atom_index_1_dist_mean_diff'] = df[f'molecule_atom_index_1_dist_mean'] - df['dist']
df[f'molecule_atom_index_1_dist_min'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('min')
 df[f'molecule_atom_index_1_dist_min_diff'] = df[f'molecule_atom_index_1_dist_min'] - df['dist']
df[f'molecule_atom_index_1_dist_std'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('std')
 df[f'molecule_type_dist_mean'] = df.groupby(['molecule_name', 'type'])['dist'].transform('mean')
 df[f'molecule_type_dist_mean_diff'] = df[f'molecule_type_dist_mean'] - df['dist']
return df
def dummies(df, list_cols):
for col in list_cols:
df_dummies = pd.get_dummies(df[col], drop_first=True,
prefix=(str(col)))
 df = pd.concat([df, df_dummies], axis=1)
 return df
def add_qm9_features(df):
data_qm9 = pd.read_pickle('.. /input/quantum-machine-9-qm9/data.covs.pickle')
 to_drop = ['type',
'linear',
'atom_index_0',
'atom_index_1',
'scalar_coupling_constant',
'U', 'G', 'H',
'mulliken_mean', 'r2', 'U0']
data_qm9 = data_qm9.drop(columns = to_drop, axis=1)
 data_qm9 = reduce_mem_usage(data_qm9,verbose=False)
 df = pd.merge(df, data_qm9, how='left', on=['molecule_name','id'])
 del data_qm9
df = dummies(df, ['type', 'atom_1'])
 return df
def get_features(df, struct):
for atom_idx in [0,1]:
df = map_atom_info(df, struct, atom_idx)
 df = df.rename(columns={'atom': f'atom_{atom_idx}',
'x': f'x_{atom_idx}',
'y': f'y_{atom_idx}',
'z': f'z_{atom_idx}'})
 struct['c_x'] = struct.groupby('molecule_name')['x'].transform('mean')
 struct['c_y'] = struct.groupby('molecule_name')['y'].transform('mean')
 struct['c_z'] = struct.groupby('molecule_name')['z'].transform('mean')
 df = find_dist(df)
 df = find_closest_atom(df)
 df = add_cos_features(df)
 df = add_dist_features(df)
 df = add_qm9_features(df)
 return df
y, lab_encoder = prepare_labels(train_df['Id'] )<prepare_x_and_y>
good_columns = ['type',
'dist_C_0_x',
'dist_C_1_x',
'dist_C_2_x',
'dist_C_3_x',
'dist_C_4_x',
'dist_F_0_x',
'dist_F_1_x',
'dist_F_2_x',
'dist_F_3_x',
'dist_F_4_x',
'dist_H_0_x',
'dist_H_1_x',
'dist_H_2_x',
'dist_H_3_x',
'dist_H_4_x',
'dist_N_0_x',
'dist_N_1_x',
'dist_N_2_x',
'dist_N_3_x',
'dist_N_4_x',
'dist_O_0_x',
'dist_O_1_x',
'dist_O_2_x',
'dist_O_3_x',
'dist_O_4_x',
'dist_C_0_y',
'dist_C_1_y',
'dist_C_2_y',
'dist_C_3_y',
'dist_C_4_y',
'dist_F_0_y',
'dist_F_1_y',
'dist_F_2_y',
'dist_F_3_y',
'dist_F_4_y',
'dist_H_0_y',
'dist_H_1_y',
'dist_H_2_y',
'dist_H_3_y',
'dist_H_4_y',
'dist_N_0_y',
'dist_N_1_y',
'dist_N_2_y',
'dist_N_3_y',
'dist_N_4_y',
'dist_O_0_y',
'dist_O_1_y',
'dist_O_2_y',
'dist_O_3_y',
'dist_O_4_y',
'dist_inv2',
'distance_closest_0',
'distance_closest_1',
'distance_farthest_0',
'distance_farthest_1',
'cos_c0_c1', 'cos_f0_f1','cos_c0_f0', 'cos_c1_f1',
'cos_center0_center1', 'cos_c0', 'cos_c1', 'cos_f0', 'cos_f1',
'cos_center0', 'cos_center1',
'molecule_atom_index_0_dist_mean',
'molecule_atom_index_0_dist_mean_diff',
'molecule_atom_index_0_dist_min',
'molecule_atom_index_0_dist_min_diff',
'molecule_atom_index_0_dist_std',
'molecule_atom_index_1_dist_mean',
'molecule_atom_index_1_dist_mean_diff',
'molecule_atom_index_1_dist_min',
'molecule_atom_index_1_dist_min_diff',
'molecule_atom_index_1_dist_std',
'molecule_type_dist_mean',
'molecule_type_dist_mean_diff',
'rc_A',
'rc_B',
'rc_C',
'mu',
'alpha',
'homo',
'lumo',
'gap',
'zpve',
'Cv',
'freqs_min',
'freqs_max',
'freqs_mean',
'mulliken_min',
'mulliken_max',
'mulliken_atom_0',
'mulliken_atom_1']
giba_columns = ['inv_dist0',
'inv_dist1',
'inv_distP',
'inv_dist0R',
'inv_dist1R',
'inv_distPR',
'inv_dist0E',
'inv_dist1E',
'inv_distPE',
'linkM0',
'linkM1',
'min_molecule_atom_0_dist_xyz',
'mean_molecule_atom_0_dist_xyz',
'max_molecule_atom_0_dist_xyz',
'sd_molecule_atom_0_dist_xyz',
'min_molecule_atom_1_dist_xyz',
'mean_molecule_atom_1_dist_xyz',
'max_molecule_atom_1_dist_xyz',
'sd_molecule_atom_1_dist_xyz',
'coulomb_C.x',
'coulomb_F.x',
'coulomb_H.x',
'coulomb_N.x',
'coulomb_O.x',
'yukawa_C.x',
'yukawa_F.x',
'yukawa_H.x',
'yukawa_N.x',
'yukawa_O.x',
'vander_C.x',
'vander_F.x',
'vander_H.x',
'vander_N.x',
'vander_O.x',
'coulomb_C.y',
'coulomb_F.y',
'coulomb_H.y',
'coulomb_N.y',
'coulomb_O.y',
'yukawa_C.y',
'yukawa_F.y',
'yukawa_H.y',
'yukawa_N.y',
'yukawa_O.y',
'vander_C.y',
'vander_F.y',
'vander_H.y',
'vander_N.y',
'vander_O.y',
'distC0',
'distH0',
'distN0',
'distC1',
'distH1',
'distN1',
'adH1',
'adH2',
'adH3',
'adH4',
'adC1',
'adC2',
'adC3',
'adC4',
'adN1',
'adN2',
'adN3',
'adN4',
'NC',
'NH',
'NN',
'NF',
'NO']
class WhaleDataset(Dataset):
def __init__(self, datafolder, datatype='train', df=None, df_bbox = None, transform=None, y=None):
self.datafolder = datafolder
self.datatype = datatype
self.y = y
if self.datatype == 'train':
self.df = df.values
self.df_bbox = df_bbox
if self.datatype == 'train':
self.image_files_list = list(df.Image)
 else:
self.image_files_list = [s for s in os.listdir(datafolder)]
self.transform = transform
def __len__(self):
return len(self.image_files_list)
 def __getitem__(self, idx):
if self.datatype == 'train':
img_name = os.path.join(self.datafolder, self.df[idx][0])
 bbox = self.df_bbox.loc[self.df[idx][0]]
x0, y0, x1, y1 = int(bbox['x0']), int(bbox['y0']), int(bbox['x1']), int(bbox['y1'])
 label = self.y[idx]
elif self.datatype == 'test':
img_name = os.path.join(self.datafolder, self.image_files_list[idx])
 bbox = self.df_bbox.loc[self.image_files_list[idx]]
x0, y0, x1, y1 = int(bbox['x0']), int(bbox['y0']), int(bbox['x1']), int(bbox['y1'])
 label = np.zeros(( NUM_CLASSES,))
img = cv2.imread(img_name)
 img = img[y0:y1, x0:x1]
image = self.transform(image = img)['image']
if self.datatype == 'train':
return image, label
elif self.datatype == 'test':
return image, label, self.image_files_list[idx]<load_from_csv>
lbl = LabelEncoder()
lbl.fit(list(train['type'].values)+ list(test['type'].values))
train['type'] = lbl.transform(list(train['type'].values))
test['type'] = lbl.transform(list(test['type'].values))
bbox_df_train = pd.read_csv(BBOX_TRAIN ).set_index('Image')
 bbox_df_test = pd.read_csv(BBOX_TEST ).set_index('Image' )<filter>
n_fold = 3
seed = round(time.time())
 folds = KFold(n_splits=n_fold, shuffle=True, random_state=seed)
 params = {'num_leaves': 400,
'objective': 'huber',
'max_depth': 9,
'learning_rate': 0.12,
"boosting_type": "gbdt",
"subsample_freq": 1,
"subsample": 0.8,
"metric": 'mae',
"verbosity": -1,
'lambda_l1': 0.8,
'lambda_l2': 0.2,
'feature_fraction': 0.6,
}
bbox = bbox_df_train.loc['0000e88ab.jpg']<prepare_x_and_y>
X_short = pd.DataFrame({'ind': list(train.index),
'type': train['type'].values,
'oof': [0] * len(train),
'target': y.values})
 X_short_test = pd.DataFrame({'ind': list(test.index),
'type': test['type'].values,
'prediction': [0] * len(test)} )
train_dataset = WhaleDataset(
 datafolder='.. /input/humpback-whale-identification/train/',
datatype='train',
df=train_df, df_bbox = bbox_df_train,
transform=data_transforms,
y=y 
)
 test_set = WhaleDataset(
 datafolder='.. /input/humpback-whale-identification/test/',
datatype='test', df_bbox = bbox_df_test,
transform=data_transforms_test
)<load_pretrained>
%%time
CV_score = 0
n_estimators = [5000, 2500, 3000, 2500, 2500, 3000, 2500, 2500]
for t in train['type'].unique() :
type_ = lbl.inverse_transform([t])[0]
print(f'
Training of type {t}: {type_}.')
 index_type =(train['type'] == t)
 index_type_test =(test['type'] == t)
 X_t = train.loc[index_type].copy()
X_test_t = test.loc[index_type_test].copy()
y_t = y[index_type]
print(f'Generating features...')
 start_time = time.time()
X_t = get_features(X_t, structures.copy())
 X_t = X_t[good_columns].fillna(0.0)
 X_test_t = get_features(X_test_t, structures.copy())
 X_test_t = X_test_t[good_columns].fillna(0.0)
 rows_to_exclude = np.where(index_type==False)[0]+1
rows_to_exclude_test = np.where(index_type_test==False)[0]+1
train_giba_t = pd.read_csv('.. /input/giba-molecular-features/train_giba.csv/train_giba.csv',
header=0, skiprows=rows_to_exclude, usecols=giba_columns)
 test_giba_t = pd.read_csv('.. /input/giba-molecular-features/test_giba.csv/test_giba.csv',
header=0, skiprows=rows_to_exclude_test, usecols=giba_columns)
 X_t = pd.concat(( X_t, train_giba_t), axis=1)
 X_test_t = pd.concat(( X_test_t,test_giba_t), axis=1)
 del train_giba_t, test_giba_t
gc.collect()
print(f'Done in {(time.time() - start_time):.2f} seconds for {X_t.shape[1]} features.')
 result_dict_lgb = train_model_regression(X=X_t, X_test=X_test_t,
y=y_t, params=params,
folds=folds,
model_type='lgb',
eval_metric='mae',
plot_feature_importance=False,
verbose=2000, early_stopping_rounds=200,
n_estimators=n_estimators[t])
 del X_t, X_test_t
gc.collect()
X_short.loc[X_short['type'] == t, 'oof'] = result_dict_lgb['oof']
X_short_test.loc[X_short_test['type'] == t, 'prediction'] = result_dict_lgb['prediction']
CV_score += np.log(np.array(result_dict_lgb['scores'] ).mean())/8
batch_size = 32
num_workers = 0
train_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers, pin_memory=True)
 test_loader = DataLoader(test_set, batch_size=batch_size, num_workers=num_workers, pin_memory=True )<choose_model_class>
sub['scalar_coupling_constant'] = X_short_test['prediction']
today = str(datetime.date.today())
 sub.to_csv(f'LGB_{today}_{CV_score:.4f}.csv', index=False )