Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<train_model>
warnings.filterwarnings("ignore")
 warnings.filterwarnings(action="ignore",category=DeprecationWarning)
 warnings.filterwarnings(action="ignore",category=FutureWarning)
n_folds = 5
folds = KFold(n_splits=n_folds)
 columns = train.columns
y_preds = np.zeros(test.shape[0])
 for fold_n,(train_index, valid_index)in enumerate(folds.split(train, y)) :
X_train, X_valid = train[columns].iloc[train_index], train[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
temp_train = lgbm.Dataset(X_train, label=y_train)
 temp_valid = lgbm.Dataset(X_valid, label=y_valid)
 clf = lgbm.train(params,temp_train, 10000, valid_sets = [temp_train, temp_valid],
verbose_eval=200, early_stopping_rounds=500)
 y_pred_valid = clf.predict(X_valid)
 print("AUC: ",roc_auc_score(y_valid, y_pred_valid))
y_preds += clf.predict(test)/ n_folds
del X_train, X_valid, y_train, y_valid
gc.collect()<save_to_csv>
DATA_PATH = '.. /input/champs-scalar-coupling'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
submission = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv')
 submission['isFraud'] = y_preds
submission.to_csv('submission.csv', index=False )<define_variables>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )
imp_feature = ['uid3_count_dist', 'feat1', 'TransactionAmt', 'feat2', 'card1_count_dist', 'card2_count_dist', 'addr1_count_dist', 'uid_count_dist', 'uid2_count_dist', 'Transaction_hour', 'uid1_count_dist', 'P_emaildomain_count_dist', 'M_na', 'M_na_count_dist', 'D15_count_dist', 'Transaction_day_of_week', 'D1', 'card5_count_dist', 'dist1_count_dist', 'C13', 'D10_count_dist', 'id_20_count_dist', 'D4_count_dist', 'id_19_count_dist', 'C13_count_dist', 'D2_count_dist', 'id_31_count_dist', 'D1_count_dist', 'DeviceInfo_count_dist', 'R_emaildomain_count_dist', 'C1', 'uid4_count_dist', 'id_33_count_dist', 'V310_count_dist', 'C2', 'C1_count_dist', 'V307_count_dist', 'C14', 'C6', 'C2_count_dist', 'C14_count_dist', 'C6_count_dist', 'id_30_count_dist', 'D3_count_dist', 'V313_count_dist', 'D11_count_dist', 'D5_count_dist', 'id_05_count_dist', 'D9_count_dist', 'C11', 'dist2_count_dist', 'C9', 'V315_count_dist', 'id_13_count_dist', 'C9_count_dist', 'id_01_count_dist', 'V130_count_dist', 'M4_count_dist', 'C11_count_dist', 'V314_count_dist', 'M6_count_dist', 'M5_count_dist', 'card4_count_dist', 'V127_count_dist', 'V312_count_dist', 'D14_count_dist', 'V308_count_dist', 'C5', 'V283_count_dist', 'id_14_count_dist', 'V83_count_dist', 'M3_count_dist', 'C5_count_dist', 'V62_count_dist', 'id_18_count_dist', 'ProductCD_count_dist', 'V87_count_dist', 'V282_count_dist', 'V317_count_dist', 'D6_count_dist', 'card6_count_dist', 'V285_count_dist', 'V76_count_dist', 'card3_count_dist', 'id_02_count_dist', 'V45_count_dist', 'V143_count_dist', 'V38_count_dist', 'V131_count_dist', 'C10'] 
<set_options>
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv' )
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline<load_from_csv>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )
sub = pd.read_csv(".. /input/ieee-fraud-detection/sample_submission.csv" )<load_from_csv>
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )
train_id = pd.read_csv(".. /input/ieee-fraud-detection/train_identity.csv")
 train_tr = pd.read_csv(".. /input/ieee-fraud-detection/train_transaction.csv" )<set_options>
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )<load_from_csv>
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df
test_id = pd.read_csv(".. /input/ieee-fraud-detection/test_identity.csv")
 test_tr = pd.read_csv(".. /input/ieee-fraud-detection/test_transaction.csv" )<merge>
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df
train = pd.merge(train_tr, train_id, on='TransactionID', how='left')
 test = pd.merge(test_tr, test_id, on='TransactionID', how='left')
 del test_id, test_tr
del train_id, train_tr
gc.collect()<randomize_order>
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df
train_pos = train[train['isFraud']==1]
train_neg = train[train['isFraud']==0]
train_neg = train_neg.sample(2*int(train_pos.shape[0]), random_state=42)
 train = pd.concat([train_pos,train_neg] ).sort_index()<define_variables>
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))
l = 3*int(train_pos.shape[0] )<drop_column>
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )
del train_pos
del train_neg<count_missing_values>
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )
train.isna().sum()<sort_values>
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=15):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full
train = train.sort_values('TransactionDT' )<remove_duplicates>
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]
train.drop_duplicates(inplace=True )<prepare_x_and_y>
def create_nn_model(input_shape):
inp = Input(shape=(input_shape,))
x = Dense(2048, activation="relu" )(inp)
 x = BatchNormalization()(x)
 x = Dense(1024, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(1024, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation="relu" )(x)
 x = BatchNormalization()(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inp, outputs=[out])
 return model
target = train["isFraud"]
train.drop(["isFraud"], axis=1, inplace=True )<define_variables>
config = tf.ConfigProto(device_count = {'GPU': 1 , 'CPU': 2})
 config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.6
sess = tf.Session(config=config)
 K.set_session(sess )
useful_features = [col for col in train.columns]<count_missing_values>
mol_types=train_csv["type"].unique()
cv_score=[]
cv_score_total=0
epoch_n = 1000
verbose = 1
batch_size = 2048
retrain =True
start_time=datetime.now()
test_prediction=np.zeros(len(test_csv))
input_features = ['atom_2', 'atom_3', 'atom_4', 'atom_5', 'atom_6', 'atom_7',
'atom_8','atom_9', 'atom_10','d_1_0', 'd_2_0', 'd_2_1', 'd_3_0',
'd_3_1', 'd_3_2', 'd_4_0', 'd_4_1', 'd_4_2', 'd_4_3', 'd_5_0',
'd_5_1', 'd_5_2', 'd_5_3', 'd_6_0', 'd_6_1', 'd_6_2', 'd_6_3',
'd_7_0', 'd_7_1', 'd_7_2', 'd_7_3', 'd_8_0', 'd_8_1', 'd_8_2',
'd_8_3', 'd_9_0', 'd_9_1', 'd_9_2', 'd_9_3', 'd_10_0', 'd_10_1', 'd_10_2',
'd_10_3']
for mol_type in mol_types:
model_name_wrt =('/kaggle/working/molecule_model_%s.hdf5' % mol_type)
 print('Training %s' % mol_type, 'out of', mol_types, '
')
 full = build_couple_dataframe(train_csv, structures_csv, mol_type, n_atoms=11)
 full2 = build_couple_dataframe(test_csv, structures_csv, mol_type, n_atoms=11)
 df_train_ = take_n_atoms(full, 11)
 df_test_ = take_n_atoms(full2, 11)
 df_train_ = df_train_.fillna(0)
 df_test_ = df_test_.fillna(0)
 input_data=StandardScaler().fit_transform(pd.concat([df_train_.loc[:,input_features],df_test_.loc[:,input_features]]))
target_data=df_train_.loc[:,"scalar_coupling_constant"].values
train_index, cv_index = train_test_split(np.arange(len(df_train_)) ,random_state=111, test_size=0.1)
 train_target=target_data[train_index]
cv_target=target_data[cv_index]
train_input=input_data[train_index]
cv_input=input_data[cv_index]
test_input=input_data[len(df_train_):,:]
nn_model=create_nn_model(train_input.shape[1])
 if not retrain:
nn_model = load_model(model_name_rd)
 nn_model.compile(loss='mae', optimizer=Adam())
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=40,verbose=1, mode='auto', restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.08,patience=28, min_lr=1e-6, mode='auto', verbose=1)
 sv_mod = callbacks.ModelCheckpoint(model_name_wrt, monitor='val_loss', save_best_only=True, period=1)
 history = nn_model.fit(train_input,[train_target],
validation_data=(cv_input,[cv_target]),
callbacks=[es, rlr, sv_mod], epochs=epoch_n, batch_size=batch_size, verbose=verbose)
 cv_predict=nn_model.predict(cv_input)
 plot_history(history, mol_type)
 accuracy=np.mean(np.abs(cv_target-cv_predict[:,0]))
print(np.log(accuracy))
cv_score.append(np.log(accuracy))
cv_score_total+=np.log(accuracy)
 test_predict=nn_model.predict(test_input)
 test_prediction[test_csv["type"]==mol_type]=test_predict[:,0]
K.clear_session()
cv_score_total/=len(mol_types)
train.isna().sum()<set_options>
print('Total training time: ', datetime.now() - start_time)
 i=0
for mol_type in mol_types:
print(mol_type,": cv score is ",cv_score[i])
 i+=1
print("total cv score is",cv_score_total )
<split><EOS> 
def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv("/kaggle/working/submission.csv", index=False)
 submits(test_prediction )