Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<count_values>
warnings.filterwarnings("ignore")
 warnings.filterwarnings(action="ignore",category=DeprecationWarning)
 warnings.filterwarnings(action="ignore",category=FutureWarning)
def clearRare(columnname, limit = 1000):
vc = test[columnname].value_counts()
common = vc > limit
common = set(common.index[common].values)
 print("Set", sum(vc <= limit), columnname, "categories to 'other';", end=" ")
 train.loc[train[columnname].map(lambda x: x not in common), columnname] = 'other'
test.loc[test[columnname].map(lambda x: x not in common), columnname] = 'other'
print("now there are", train[columnname].nunique() , "categories in train" )<train_model>
DATA_PATH = '.. /input/champs-scalar-coupling'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )<drop_column>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )
clearRare("device.browser")
 clearRare("device.operatingSystem")
 clearRare("geoNetwork.country")
 clearRare("geoNetwork.city")
 clearRare("geoNetwork.metro")
 clearRare("geoNetwork.networkDomain")
 clearRare("geoNetwork.region")
 clearRare("geoNetwork.subContinent")
 clearRare("trafficSource.adContent")
 clearRare("trafficSource.campaign")
 clearRare("trafficSource.keyword")
 clearRare("trafficSource.medium")
 clearRare("trafficSource.referralPath")
 clearRare("trafficSource.source" )<data_type_conversions>
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv' )
for df in [train, test]:
df["Avg.Session Duration"][df["Avg.Session Duration"] == 0] = "00:00:00"
df["Avg.Session Duration"] = df["Avg.Session Duration"].str.split(':' ).apply(lambda x: int(x[0])* 60 + int(x[1]))
df["Bounce Rate"] = df["Bounce Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float )<feature_engineering>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )
for df in [train, test]:
df["id_incoherence"] = pd.to_datetime(df.visitId, unit='s')!= df.date
df["visitId_dublicates"] = df.visitId.map(df.visitId.value_counts())
 df["session_dublicates"] = df.sessionId.map(df.sessionId.value_counts() )<feature_engineering>
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )
for df in [train, test]:
df['weekday'] = df['date'].dt.dayofweek.astype(object)
 df['time'] = df['date'].dt.second + df['date'].dt.minute*60 + df['date'].dt.hour*3600
df['day'] = df['date'].dt.date<data_type_conversions>
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures
df = pd.concat([train, test])
 df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
 df['prev_session'] =(df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)).astype(np.int64)// 1e9 // 60 // 60
df['next_session'] =(df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)).astype(np.int64)// 1e9 // 60 // 60
df.sort_index(inplace=True)
 train = df[:len(train)]
test = df[len(train):]<feature_engineering>
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df
for df in [train, test]:
df['source.country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']
df['campaign.medium'] = df['trafficSource.campaign'] + '_' + df['trafficSource.medium']
df['browser.category'] = df['device.browser'] + '_' + df['device.deviceCategory']
df['browser.os'] = df['device.browser'] + '_' + df['device.operatingSystem']<feature_engineering>
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df
for df in [train, test]:
df['device_deviceCategory_channelGrouping'] = df['device.deviceCategory'] + "_" + df['channelGrouping']
df['channelGrouping_browser'] = df['device.browser'] + "_" + df['channelGrouping']
df['channelGrouping_OS'] = df['device.operatingSystem'] + "_" + df['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[i + "_" + j] = df[i] + "_" + df[j]
df['content.source'] = df['trafficSource.adContent'].astype(str)+ "_" + df['source.country']
df['medium.source'] = df['trafficSource.medium'] + "_" + df['source.country']<feature_engineering>
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df
for feature in ["totals.hits", "totals.pageviews"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].mean()
train["usermean_" + feature] = train.fullVisitorId.map(info)
 test["usermean_" + feature] = test.fullVisitorId.map(info)
 for feature in ["visitNumber"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].max()
train["usermax_" + feature] = train.fullVisitorId.map(info)
 test["usermax_" + feature] = test.fullVisitorId.map(info )<define_variables>
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime',
'month', 'day', 'help']
cat_cols = [f for f in train.columns if(train[f].dtype == 'object' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]<count_unique_values>
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )
train[cat_cols].nunique()<data_type_conversions>
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )
for col in cat_cols:
lbl = LabelEncoder()
lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))
 train[col] = lbl.transform(list(train[col].values.astype('str')))
 test[col] = lbl.transform(list(test[col].values.astype('str')) )<data_type_conversions>
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=15):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full
for col in real_cols:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float )<drop_column>
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]
for to_del in ["date", "sessionId", "visitId", "day"]:
del train[to_del]
del test[to_del]<define_variables>
def create_nn_model(input_shape):
inp = Input(shape=(input_shape,))
x = Dense(2048, activation="relu" )(inp)
 x = BatchNormalization()(x)
 x = Dropout(0.1 )(x)
 x = Dense(1024, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation="relu" )(x)
 x = BatchNormalization()(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inp, outputs=[out])
 return model
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime', "month", "help"]
cat_cols = [f for f in train.columns if(train[f].dtype == 'int64' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]<compute_test_metric>
config = tf.ConfigProto(device_count = {'GPU': 1 , 'CPU': 2})
 config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.6
sess = tf.Session(config=config)
 K.set_session(sess )
def score(data, y):
validation_res = pd.DataFrame(
 {"fullVisitorId": data["fullVisitorId"].values,
"transactionRevenue": data["totals.transactionRevenue"].values,
"predictedRevenue": np.expm1(y)})
 validation_res = validation_res.groupby("fullVisitorId")["transactionRevenue", "predictedRevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionRevenue"].values),
np.log1p(validation_res["predictedRevenue"].values)) )<categorify>
mol_types=train_csv["type"].unique()
cv_score=[]
cv_score_total=0
epoch_n = 2000
verbose = 1
batch_size = 2048
retrain =True
start_time=datetime.now()
test_prediction=np.zeros(len(test_csv))
distance_features = [
'd_1_0', 'd_2_0', 'd_2_1', 'd_3_0',
'd_3_1', 'd_3_2', 'd_4_0', 'd_4_1', 'd_4_2', 'd_4_3', 'd_5_0',
'd_5_1', 'd_5_2', 'd_5_3', 'd_6_0', 'd_6_1', 'd_6_2', 'd_6_3',
'd_7_0', 'd_7_1', 'd_7_2', 'd_7_3', 'd_8_0', 'd_8_1', 'd_8_2',
'd_8_3', 'd_9_0', 'd_9_1', 'd_9_2', 'd_9_3', 'd_10_0', 'd_10_1', 'd_10_2',
'd_10_3'
]
atom_features = [
'atom_2', 'atom_3', 'atom_4', 'atom_5', 'atom_6', 'atom_7', 'atom_8','atom_9', 'atom_10'
]
input_features = atom_features + distance_features
class FeatureTransformer:
def transform(self, dataset, ohe_features=[], continuous_features=[]):
ohe_df = OneHotEncoder().fit_transform(dataset.loc[:, ohe_features] ).toarray()
skews = dataset.loc[:, continuous_features].skew().to_dict()
for column, skew in skews.items() :
if skew > 1:
dataset[column] = np.log1p(dataset[column])
 return np.concatenate([StandardScaler().fit_transform(dataset.loc[:, continuous_features]), ohe_df], axis=1)
 for mol_type in mol_types:
model_name_wrt =('/kaggle/working/molecule_model_%s.hdf5' % mol_type)
 print('Training %s' % mol_type, 'out of', mol_types, '
')
 full = build_couple_dataframe(train_csv, structures_csv, mol_type, n_atoms=11)
 full2 = build_couple_dataframe(test_csv, structures_csv, mol_type, n_atoms=11)
 df_train_ = take_n_atoms(full, 11)
 df_test_ = take_n_atoms(full2, 11)
 df_train_ = df_train_.fillna(0)
 df_test_ = df_test_.fillna(0)
 input_data = FeatureTransformer().transform(
 dataset=pd.concat([df_train_.loc[:,input_features], df_test_.loc[:,input_features]]),
ohe_features=atom_features,
continuous_features=distance_features 
)
 target_data=df_train_.loc[:,"scalar_coupling_constant"].values
train_index, cv_index = train_test_split(np.arange(len(df_train_)) ,random_state=111, test_size=0.1)
 train_target=target_data[train_index]
cv_target=target_data[cv_index]
train_input=input_data[train_index]
cv_input=input_data[cv_index]
test_input=input_data[len(df_train_):,:]
nn_model=create_nn_model(train_input.shape[1])
 if not retrain:
nn_model = load_model(model_name_rd)
 nn_model.compile(loss='mae', optimizer=Adam())
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=50, verbose=1, mode='auto', restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=30, min_lr=1e-6, mode='auto', verbose=1)
 sv_mod = callbacks.ModelCheckpoint(model_name_wrt, monitor='val_loss', save_best_only=True, period=1)
 history = nn_model.fit(train_input, [train_target],
validation_data=(cv_input, [cv_target]),
callbacks=[es, rlr, sv_mod], epochs=epoch_n, batch_size=batch_size, verbose=verbose)
 cv_predict=nn_model.predict(cv_input)
 plot_history(history, mol_type)
 accuracy=np.mean(np.abs(cv_target-cv_predict[:,0]))
print(np.log(accuracy))
cv_score.append(np.log(accuracy))
cv_score_total+=np.log(accuracy)
 test_predict=nn_model.predict(test_input)
 test_prediction[test_csv["type"]==mol_type]=test_predict[:,0]
K.clear_session()
cv_score_total/=len(mol_types)
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['fullVisitorId'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 50, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["totals.transactionRevenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["totals.transactionRevenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score<choose_model_class>
print('Total training time: ', datetime.now() - start_time)
 i=0
for mol_type in mol_types:
print(mol_type,": cv score is ",cv_score[i])
 i+=1
print("total cv score is",cv_score_total )
<choose_model_class><EOS> 
def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv("/kaggle/working/submission.csv", index=False)
 submits(test_prediction )