Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<feature_engineering>
%matplotlib inline
train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24 
<data_type_conversions>
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
train['uid'] = train['card1'].astype(str)+'_'+train['card2'].astype(str)
 test['uid'] = test['card1'].astype(str)+'_'+test['card2'].astype(str)
 train['uid1'] = train['uid'].astype(str)+'_'+train['card3'].astype(str)
 test['uid1'] = test['uid'].astype(str)+'_'+test['card3'].astype(str)
 train['uid2'] = train['uid'].astype(str)+'_'+train['card3'].astype(str)+'_'+train['card5'].astype(str)
 test['uid2'] = test['uid'].astype(str)+'_'+test['card3'].astype(str)+'_'+test['card5'].astype(str)
 train['uid3'] = train['uid2'].astype(str)+'_'+train['addr1'].astype(str)+'_'+train['addr2'].astype(str)
 test['uid3'] = test['uid2'].astype(str)+'_'+test['addr1'].astype(str)+'_'+test['addr2'].astype(str)
 train['uid4'] = train['card4'].astype(str)+'_'+train['card6'].astype(str)
 test['uid4'] = test['card4'].astype(str)+'_'+test['card6'].astype(str)
 train['TransactionAmt_check'] = np.where(train['TransactionAmt'].isin(test['TransactionAmt']), 1, 0)
 test['TransactionAmt_check'] = np.where(test['TransactionAmt'].isin(train['TransactionAmt']), 1, 0)
<concatenate>
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 120)
 pd.set_option('display.max_columns', 120 )
train = pd.concat([train, test] )<data_type_conversions>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )
train['M_na'] = abs(train.isna().sum(axis=1 ).astype(np.int8))<feature_engineering>
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = 'Others'
gc.collect()
return dataframe<data_type_conversions>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int )<define_variables>
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )
neglect = ["TransactionAmt", 'Transaction_day_of_week', 'Transaction_hour', "TransactionAmt_Log", 'TransactionAmt_decimal']<define_variables>
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other',
'optonline.net': 'other', 'hotmail.co.uk': 'microsoft', 'comcast.net': 'other',
'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo', 'yahoo.es': 'yahoo',
'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol',
'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink', 'gmail.com': 'google',
'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other', 'web.de': 'other',
'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo', 'servicios-ta.com': 'other',
'netzero.net': 'other', 'suddenlink.net': 'other', 'roadrunner.com': 'other',
'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att',
'frontier.com': 'yahoo', 'anonymous.com': 'other', 'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo', 'ymail.com': 'yahoo',
'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft',
'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']<feature_engineering>
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<define_variables>
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df
useful_features = [col for col in train.columns if col not in neglect]<count_values>
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df
<define_variables>
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))
dropping =["D8_count_dist", "V138_count_dist", "V139_count_dist", "V140_count_dist", "V141_count_dist",\
"V146_count_dist", "V147_count_dist", "V148_count_dist", "V149_count_dist", "V144_count_dist",\
"V145_count_dist", "V150_count_dist", "V151_count_dist", "V152_count_dist", "V153_count_dist",\
"V154_count_dist", "V155_count_dist", "V156_count_dist", "V157_count_dist", "V158_count_dist",\
"V159_count_dist", "V160_count_dist", "V161_count_dist", "V162_count_dist", "V163_count_dist",\
"V164_count_dist", "V165_count_dist", "V166_count_dist", "V168_count_dist", "V170_count_dist",\
"V171_count_dist", "V172_count_dist", "V173_count_dist", "V174_count_dist", "V175_count_dist",\
"V176_count_dist", "V177_count_dist", "V178_count_dist", "V179_count_dist", "V180_count_dist",\
"V181_count_dist", "V182_count_dist", "V183_count_dist", "V184_count_dist", "V185_count_dist",\
"V186_count_dist", "V187_count_dist", "V188_count_dist", "V189_count_dist", "V190_count_dist",\
"V191_count_dist", "V192_count_dist", "V193_count_dist", "V194_count_dist", "V195_count_dist",\
"V196_count_dist", "V197_count_dist", "V198_count_dist", "V199_count_dist", "V200_count_dist",\
"V201_count_dist", "V202_count_dist", "V203_count_dist", "V204_count_dist", "V205_count_dist",\
"V206_count_dist", "V207_count_dist", "V208_count_dist", "V209_count_dist", "V210_count_dist",\
"V211_count_dist", "V212_count_dist", "V213_count_dist", "V214_count_dist", "V215_count_dist",\
"V216_count_dist", "V218_count_dist", "V219_count_dist", "V221_count_dist", "V222_count_dist",\
"V223_count_dist", "V224_count_dist", "V225_count_dist", "V226_count_dist", "V227_count_dist",\
"V228_count_dist", "V229_count_dist", "V230_count_dist", "V231_count_dist", "V232_count_dist",\
"V233_count_dist", "V234_count_dist", "V235_count_dist", "V236_count_dist", "V237_count_dist",\
"V205_count_dist", "V205_count_dist", "V205_count_dist", "V205_count_dist", "V205_count_dist",\
"V238_count_dist", "V239_count_dist", "V240_count_dist", "V241_count_dist", "V242_count_dist",\
"V243_count_dist", "V244_count_dist", "V245_count_dist","V246_count_dist", "V247_count_dist",\
"V248_count_dist", "V249_count_dist", "V250_count_dist", "V251_count_dist", "V252_count_dist",\
"V253_count_dist", "V254_count_dist", "V255_count_dist", "V256_count_dist", "V257_count_dist",\
"V258_count_dist", "V259_count_dist", "V260_count_dist", "V261_count_dist", "V262_count_dist",\
"V263_count_dist", "V264_count_dist", "V265_count_dist", "V266_count_dist", "V267_count_dist",\
"V268_count_dist", "V269_count_dist", "V270_count_dist", "V271_count_dist", "V272_count_dist",\
"V273_count_dist", "V274_count_dist", "V275_count_dist", "V276_count_dist", "V277_count_dist",\
"V278_count_dist", "V323_count_dist", "V324_count_dist", "V325_count_dist", "V326_count_dist",\
"V327_count_dist", "V328_count_dist", "V329_count_dist", "V330_count_dist", "V331_count_dist",\
"V332_count_dist", "V333_count_dist", "V334_count_dist", "V335_count_dist", "V336_count_dist",\
"V237_count_dist", "V238_count_dist", "V239_count_dist", "id_04_count_dist", "id_06_count_dist",\
"id_08_count_dist", "id_10_count_dist", "id_22_count_dist", "id_27_count_dist", "id_29_count_dist",\
"id_36_count_dist", "id_37_count_dist", "id_38_count_dist"] 
<feature_engineering>
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )
<feature_engineering>
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )
<prepare_x_and_y>
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full
<count_missing_values>
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]
train.isna().sum()<data_type_conversions>
%%time
full = build_couple_dataframe(train_csv, structures_csv, '1JHN', n_atoms=10)
 print(full.shape )
train.fillna(-333, inplace=True )<prepare_x_and_y>
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 X_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.2, random_state=128)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape
X = train.iloc[:l, :]
test = train.iloc[l:, :]<categorify>
LGB_PARAMS = {
'objective': 'regression',
'metric': 'mae',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate': 0.2,
'num_leaves': 128,
'min_child_samples': 79,
'max_depth': 9,
'subsample_freq': 1,
'subsample': 0.9,
'bagging_seed': 11,
'reg_alpha': 0.1,
'reg_lambda': 0.3,
'colsample_bytree': 1.0
}
for f in X.columns:
if X[f].dtype.name =='object' or test[f].dtype.name =='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X[f].values)+ list(test[f].values))
X[f] = lbl.transform(list(X[f].values))
test[f] = lbl.transform(list(test[f].values))<prepare_x_and_y>
model = LGBMRegressor(**LGB_PARAMS, n_estimators=1500, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=100, early_stopping_rounds=200)
 y_pred = model.predict(X_val)
 np.log(mean_absolute_error(y_val, y_pred))
y=target<define_variables>
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data
categorical_var = np.where(train.dtypes != np.float)[0]
print('
Categorical Variables indices : ',categorical_var )<drop_column>
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, n_folds=5, n_splits=5, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 cv_score = 0
if n_folds > n_splits:
n_splits = n_folds
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
if fold >= n_folds:
break
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
model = LGBMRegressor(**LGB_PARAMS, n_estimators=1500, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=100, early_stopping_rounds=200)
 y_val_pred = model.predict(X_val)
 val_score = np.log(mean_absolute_error(y_val, y_val_pred))
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(X_test)/ n_folds
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
return cv_score
del train<feature_engineering>
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
N_FOLDS = 3
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score
<init_hyperparams>
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )