Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<data_type_conversions>
%matplotlib inline
all_data['card1'] = all_data['card1'].fillna(0)
 all_data['card2'] = all_data['card2'].fillna(0)
 all_data['card3'] = all_data['card3'].fillna(0)
 all_data['card5'] = all_data['card5'].fillna(0)
 all_data['card4'] = all_data['card4'].fillna('nan')
 all_data['card6'] = all_data['card6'].fillna('nan' )<feature_engineering>
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
all_data['card_hash'] = all_data.apply(lambda x: card_info_hash(x), axis=1 )<filter>
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 120)
 pd.set_option('display.max_columns', 120 )
def get_data_by_card_hash(data, card_hash):
mask = data['card_hash']==card_hash
return data.loc[mask,:].copy()
def get_data_by_device_hash(data, device_hash):
mask = data['device_hash']==device_hash
return data.loc[mask,:].copy()
def get_data_by_card_and_startdate(data, card_hash, device_hash):
mask =(data['client_firstdate_days']==card_hash)&(data['card_hash']==device_hash)
 return data.loc[mask,:].copy()<groupby>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10)
all_data['count']=1
grp = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['count'].agg('sum')
 display_group = get_data_by_card_and_startdate(all_data,grp[grp>10].index[0][0],grp[grp>10].index[0][1])
 display_group[['DT_time','TransactionAmt','V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo','dist1','dist2','P_emaildomain']]<groupby>
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )
s = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['isFraud'].agg(['mean', 'count'] )<concatenate>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )
Test_ID=[]
for ind in tqdm(s[(s['mean']==1)].index):
very_strange_thing = get_data_by_card_and_startdate(all_data, ind[0],ind[1])
 Test_ID.extend(very_strange_thing[very_strange_thing['isFraud'].isna() ]['TransactionID'].tolist() )<save_model>
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )
np.save("test_IDs", Test_ID )<load_from_csv>
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures
submit = pd.read_csv(".. /input/ieeecis-fraud-detection-results/stack_gmean_09_16th_2019_LB0pt9530.csv")
<feature_engineering>
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df
mask = submit['TransactionID'].isin(Test_ID)
 submit.loc[mask,'isFraud'] =1<filter>
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df
submit.loc[mask,:]<save_to_csv>
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df
submit.to_csv('submit_try.csv',index = False )<set_options>
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))
warnings.filterwarnings('ignore' )<load_from_csv>
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv' )<merge>
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )
df_train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 df_test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )<drop_column>
coupling_type='1JHC'
base, structures = build_type_dataframes(train_csv, structures_csv, coupling_type )
for df in [train_transaction, train_identity, test_transaction, test_identity]:
del df<drop_column>
base = add_coordinates(base, structures, 0)
 base.head()
drop_col = ['V300','V309','V111','V124','V106','V125','V315','V134','V102','V123','V316','V113',
'V136','V305','V110','V299','V289','V286','V318','V103','V304','V116','V29','V284','V293',
'V137','V295','V301','V104','V311','V115','V109','V119','V321','V114','V133','V122','V319',
'V105','V112','V118','V117','V121','V108','V135','V320','V303','V297','V120',
'V1','V14','V41','V65','V88','V107']
for df in [df_train, df_test]:
df = df.drop(drop_col, axis=1 )<data_type_conversions>
base = add_coordinates(base, structures, 1)
 base.head()
def make_time_feature(df):
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(( df['DT'].dt.year-2017)*12 + df['DT'].dt.month ).astype(np.int8)
 df['DT_W'] =(( df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear ).astype(np.int8)
 df['DT_D'] =(( df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear ).astype(np.int16)
 df['DT_hour'] =(df['DT'].dt.hour ).astype(np.int8)
 df['DT_day_week'] =(df['DT'].dt.dayofweek ).astype(np.int8)
 df['DT_day_month'] =(df['DT'].dt.day ).astype(np.int8)
 df['is_holiday'] =(df['DT'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8)
 return df<feature_engineering>
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full
for df in [df_train, df_test]:
df = make_time_feature(df=df )<load_from_csv>
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]
df_deviceinfo = pd.read_csv('.. /input/ieee-deviceinfo/DeviceInfo.csv' )<merge>
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data
df_train = pd.merge(df_train, df_deviceinfo, on='DeviceInfo', how='left')
 df_test = pd.merge(df_test, df_deviceinfo, on='DeviceInfo', how='left' )<drop_column>
import xgboost as xgb
del df_deviceinfo<feature_engineering>
xgb_params = {'eta': 0.1,
'max_depth': 11,
'subsample': 0.9,
'objective': 'reg:linear',
'eval_metric': 'mae',
'tree_method':'gpu_hist',
'colsample_bytree':1,
'alpha':0.2,
'lambda':0.1,
}
for df in [df_train, df_test]:
df['OS'] = df['id_30'].apply(lambda x: str(x ).split(' ')[0])
 df['OS_version'] = df['id_30'].apply(lambda x: str(x ).split(' ')[-1])
 df['OS_version'] = df['OS_version'].apply(lambda x: str(x ).replace('_', '' ).replace('.', ''))
df['OS_version'] = df['OS_version'].apply(lambda x: int(x)if x.isdigit() else np.nan )<define_variables>
def train_and_predict_for_one_coupling_type(coupling_type, submission,oof_sub,n_atoms, n_folds=5,n_depth=9, n_splits=5, random_state=129):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 oof = np.zeros(X_data.shape[0],dtype='float32')
 cv_score = 0
kfold = KFold(n_splits=n_folds, shuffle=True, random_state=random_state)
 for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
train_data=xgb.DMatrix(data=X_train,label=y_train)
 vaild_data=xgb.DMatrix(data=X_val,label=y_val)
 xgb_params['max_depth']=n_depth
watchlist = [(vaild_data, 'valid_data')]
if coupling_type in ['1JHN']:
iterations=18000
if coupling_type == '1JHC':
iterations=11000
else:
iterations=13000
model = xgb.train(dtrain=train_data, num_boost_round=iterations, evals=watchlist, early_stopping_rounds=500,params=xgb_params,verbose_eval=500)
 y_val_pred=model.predict(xgb.DMatrix(X_val))
val_score = np.log(mean_absolute_error(y_val, y_val_pred))
oof[val_index]=y_val_pred
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(xgb.DMatrix(X_test)) / n_folds
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
oof_sub.loc[train_csv['type'] == coupling_type, 'scalar_coupling_constant'] = oof
return cv_score
browser_release_date = {'android browser 4.0' : '2011-10-18',
'android webview 4.0' : '2011-10-18',
'chrome 43.0 for android' : '2015-04-15',
'chrome 46.0 for android' : '2015-10-14',
'chrome 49.0' : '2016-03-02',
'chrome 49.0 for android' : '2016-03-09',
'chrome 50.0 for android' : '2016-04-26',
'chrome 51.0' : '2016-05-25',
'chrome 51.0 for android' : '2016-06-01',
'chrome 52.0 for android' : '2016-07-27',
'chrome 53.0 for android' : '2016-09-07',
'chrome 54.0 for android' : '2016-10-19',
'chrome 55.0' : '2016-12-01',
'chrome 55.0 for android' : '2016-12-06',
'chrome 56.0' : '2017-01-25',
'chrome 56.0 for android' : '2017-02-01',
'chrome 57.0' : '2017-03-09',
'chrome 57.0 for android' : '2017-03-16',
'chrome 58.0' : '2017-04-19',
'chrome 58.0 for android' : '2017-04-20',
'chrome 59.0' : '2017-06-05',
'chrome 59.0 for android' : '2017-06-06',
'chrome 60.0' : '2017-07-25',
'chrome 60.0 for android' : '2017-07-31',
'chrome 61.0' : '2017-09-05',
'chrome 61.0 for android' : '2017-09-05',
'chrome 62.0' : '2017-10-17',
'chrome 62.0 for android' : '2017-10-19',
'chrome 62.0 for ios' : '2017-10-18',
'chrome 63.0' : '2017-12-06',
'chrome 63.0 for android' : '2017-12-05',
'chrome 63.0 for ios' : '2017-12-05',
'chrome 64.0' : '2018-01-24',
'chrome 64.0 for android' : '2018-01-23',
'chrome 64.0 for ios' : '2018-01-24',
'chrome 65.0' : '2018-03-06',
'chrome 65.0 for android' : '2018-03-06',
'chrome 65.0 for ios' : '2018-03-06',
'chrome 66.0' : '2018-04-17',
'chrome 66.0 for android' : '2018-04-17',
'chrome 66.0 for ios' : '2018-04-17',
'chrome 67.0' : '2018-05-29',
'chrome 67.0 for android' : '2018-05-31',
'chrome 69.0' : '2018-09-04',
'edge 13.0' : '2015-09-15',
'edge 14.0' : '2016-02-18',
'edge 15.0' : '2016-10-07',
'edge 16.0' : '2017-09-26',
'edge 17.0' : '2018-04-30',
'firefox 47.0' : '2016-06-07',
'firefox 48.0' : '2016-08-02',
'firefox 52.0' : '2017-03-07',
'firefox 55.0' : '2017-08-08',
'firefox 56.0' : '2017-09-28',
'firefox 57.0' : '2017-11-14',
'firefox 58.0' : '2018-01-23',
'firefox 59.0' : '2018-03-13',
'firefox 60.0' : '2018-05-09',
'firefox mobile 61.0' : '2018-06-26',
'google search application 48.0' : '2016-01-20',
'google search application 49.0' : '2016-03-02',
'ie 11.0 for desktop' : '2013-10-17',
'ie 11.0 for tablet' : '2013-10-17',
'mobile safari 10.0' : '2016-09-14',
'mobile safari 11.0' : '2017-09-20',
'mobile safari 8.0' : '2014-09-17',
'mobile safari 9.0' : '2015-09-16',
'opera 49.0' : '2017-11-08',
'opera 51.0' : '2018-02-07',
'opera 52.0' : '2018-03-14',
'opera 53.0' : '2018-05-10',
'safari 10.0' : '2016-09-20',
'safari 11.0' : '2017-09-19',
'safari 9.0' : '2015-09-30',
'samsung browser 3.3' : '2015-08-01',
'samsung browser 4.0' : '2016-03-11',
'samsung browser 4.2' : '2016-08-19',
'samsung browser 5.2' : '2016-11-01',
'samsung browser 5.4' : '2017-05-01',
'samsung browser 6.2' : '2017-08-01',
'samsung browser 6.4' : '2018-02-19',
'samsung browser 7.0' : '2018-03-01',
}<data_type_conversions>
model_params = {
'1JHN': 10,
'1JHC': 13,
'2JHH': 10,
'2JHN': 10,
'2JHC': 10,
'3JHH': 10,
'3JHC': 10,
'3JHN': 10
}
model_depth=\
{
'1JHN': 11,
'1JHC': 11,
'2JHH': 11,
'2JHN': 11,
'2JHC': 11,
'3JHH': 11,
'3JHC': 11,
'3JHN': 11
}
N_FOLDS = 5
submission = submission_csv.copy()
oof_submission=train_csv[['type','scalar_coupling_constant']].copy()
oof_submission['scalar_coupling_constant']=0
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission,oof_submission,n_atoms=model_params[coupling_type], n_folds=N_FOLDS,n_depth=model_depth[coupling_type],random_state=150)
 cv_scores[coupling_type] = cv_score
def time_split(val):
try:
return datetime.datetime.strptime(str(x), '%Y-%m-%d')
 except:
return pd.NaT<feature_engineering>
oof_submission.head()
oof_submission.to_csv('xgb_oof.csv',index=False )
<define_variables><EOS> 
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )