Kaggle competition: Predicting Molecular Properties
%%time
y_dummy=X.revenue_status_sum.apply(lambda x: 0 if x==0 else 1)
 y=X.totals_transactionRevenue_sum.apply(lambda x: np.log1p(x))
X=X.drop(['totals_transactionRevenue_sum','fullVisitorId','revenue_status_sum'],axis=1)
 col=['fullVisitorId','totals_transactionRevenue_sum']
final=X_test[col]
final.columns=['fullVisitorId','PredictedLogRevenue']
X_test=X_test.drop(['fullVisitorId','totals_transactionRevenue_sum','revenue_status_sum'],axis=1 )<choose_model_class>
dtypes = {'atom_index_0':'uint8',
'atom_index_1':'uint8',
'scalar_coupling_constant':'float32',
'num_C':'uint8',
'num_H':'uint8',
'num_N':'uint8',
'num_O':'uint8',
'num_F':'uint8',
'total_atoms':'uint8',
'num_bonds':'uint8',
'num_mol_bonds':'uint8',
'min_d':'float32',
'mean_d':'float32',
'max_d':'float32',
'space_dr':'float32',
'bond_dr':'float32',
'bond_1':'uint8',
'bond_2':'uint8',
'bond_3':'uint8',
'atom_0_pc':'float32',
'atom_end_pc':'float32',
'atom_2_hyb':'uint8',
'atom_3_hyb':'uint8',
'atom_end_hyb':'uint8',
'path_count':'uint8',
'atom_0_min':'float32',
'atom_0_mean':'float32',
'atom_0_max':'float32',
'atom_0_Cmin':'float32',
'atom_0_Cmean':'float32',
'atom_0_Cmax':'float32',
'atom_0_Omin':'float32',
'atom_0_Omean':'float32',
'atom_0_Omax':'float32',
'atom_0_Nmin':'float32',
'atom_0_Nmean':'float32',
'atom_0_Nmax':'float32',
'atom_0_Fmin':'float32',
'atom_0_Fmean':'float32',
'atom_0_Fmax':'float32',
'atom_end_min':'float32',
'atom_end_mean':'float32',
'atom_end_max':'float32',
'atom_end_Cmin':'float32',
'atom_end_Cmean':'float32',
'atom_end_Cmax':'float32',
'atom_end_Omin':'float32',
'atom_end_Omean':'float32',
'atom_end_Omax':'float32',
'atom_end_Nmin':'float32',
'atom_end_Nmean':'float32',
'atom_end_Nmax':'float32',
'atom_end_Fmin':'float32',
'atom_end_Fmean':'float32',
'atom_end_Fmax':'float32',
'Dmin_COM':'float32',
'Dmean_COM':'float32',
'Dmax_COM':'float32',
'COM_dr_0': 'float32',
'COM_dr_1': 'float32',
'bond2_angle': 'float32',
'bond3_angle': 'float32'
}
%%time
model=LGBMRegressor(boosting_type='gbdt',num_leaves=31,max_depth=-1,learning_rate=0.01,n_estimators=1000,max_bin=255,subsample_for_bin=50000,
objective=None,min_split_gain=0,min_child_weight=3,min_child_samples=10,subsample=1,subsample_freq=1,colsample_bytree=1,
reg_alpha=0.1,reg_lambda=0,seed=17,silent=False,nthread=-1,n_jobs=-1)
 k=1
splits=5
avg_score=0
skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=200)
 print('
Starting KFold iterations...')
 for train_index,test_index in skf.split(X,y_dummy):
df_X=X.iloc[train_index,:]
df_y=y.iloc[train_index]
val_X=X.iloc[test_index,:]
val_y=y.iloc[test_index]
model.fit(df_X,df_y)
 preds_x=pd.Series(model.predict(val_X))
acc=rsme(val_y,preds_x)
 print('Iteration:',k,' rmse:',acc)
 if k==1:
score=acc
model1=model
preds=pd.Series(model.predict(X_test))
else:
preds1=pd.Series(model.predict(X_test))
preds=preds+preds1
if score>acc:
score=acc
model1=model
avg_score=avg_score+acc
k=k+1
print('
Best score:',score,' Avg Score:',avg_score/splits)
 preds=preds/splits<save_to_csv>
train = pd.read_csv(".. /input/predmolprop-featureengineering-final/train_extend.csv",dtype=dtypes)
 test = pd.read_csv(".. /input/predmolprop-featureengineering-finaltest/test_extend.csv",dtype=dtypes )
submission=pd.read_csv(path1+'sample_submission.csv')
 submission=submission.join(final.set_index('fullVisitorId'),on='fullVisitorId',lsuffix='_sub')
 submission.drop('PredictedLogRevenue_sub',axis=1,inplace=True)
 submission.fillna(0,inplace=True)
 submission.to_csv('LGBM_submission.csv',index=False )<set_options>
cols = ['atom_0_type2','atom_2_type','atom_3_type','atom_end_type2']
for col in cols:
enc = LabelEncoder()
train[col]=enc.fit_transform(train[col] ).astype(np.uint8)
 test[col]=enc.transform(test[col] ).astype(np.uint8)
 del cols
gc.enable()
features = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\
'visitNumber', 'visitStartTime', 'device.browser',\
'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',\
'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country',\
'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region',\
'geoNetwork.subContinent', 'totals.bounces', 'totals.hits',\
'totals.newVisits', 'totals.pageviews', 'totals.transactionRevenue',\
'trafficSource.adContent', 'trafficSource.campaign',\
'trafficSource.isTrueDirect', 'trafficSource.keyword',\
'trafficSource.medium', 'trafficSource.referralPath',\
'trafficSource.source', 'customDimensions']
def load_df(csv_path):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
ans = pd.DataFrame()
dfs = pd.read_csv(csv_path, sep=',',
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
chunksize=100000)
 for df in dfs:
df.reset_index(drop=True, inplace=True)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 use_df = df[features]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis=0 ).reset_index(drop=True)
 return ans<load_from_csv>
prefix_train= ['id', 'type', 'scalar_coupling_constant']
prefix_test= ['id', 'type']
fc_distance = ['space_dr','min_d','mean_d', 'max_d']
fc_COM = ['Dmin_COM', 'Dmean_COM', 'Dmax_COM']
fc_size = ['num_mol_bonds', 'total_atoms','num_C', 'num_H', 'num_N', 'num_O', 'num_F']
fc_atom_0 = ['atom_0_pc','atom_0_type2','COM_dr_0',
'atom_0_min','atom_0_mean', 'atom_0_max',
'atom_0_Cmin', 'atom_0_Cmean','atom_0_Cmax',
'atom_0_Nmin', 'atom_0_Nmean', 'atom_0_Nmax',
'atom_0_Omin', 'atom_0_Omean','atom_0_Omax',
'atom_0_Fmin', 'atom_0_Fmean', 'atom_0_Fmax']
fc_atom_end = ['atom_end_pc', 'atom_end_type2', 'COM_dr_1',
'atom_end_min','atom_end_mean', 'atom_end_max',
'atom_end_Cmin', 'atom_end_Cmean','atom_end_Cmax',
'atom_end_Nmin', 'atom_end_Nmean', 'atom_end_Nmax',
'atom_end_Omin', 'atom_end_Omean','atom_end_Omax',
'atom_end_Fmin', 'atom_end_Fmean', 'atom_end_Fmax']
fc_2 = ['bond_dr','path_count','atom_2_type','bond_2','bond2_angle']
fc_3 = ['atom_3_type','bond_3','bond3_angle']
fc1 = set(fc_distance+fc_COM+fc_size+fc_atom_0+fc_atom_end)
 train = train[prefix_train+fc_distance+fc_COM+fc_size+fc_atom_0+fc_atom_end+fc_2+fc_3]
test = test[prefix_test+fc_distance+fc_COM+fc_size+fc_atom_0+fc_atom_end+fc_2+fc_3]
len(train.columns )
%%time
train = load_df('.. /input/train_v2.csv')
 test = load_df('.. /input/test_v2.csv')
 print('train date:', min(train['date']), 'to', max(train['date']))
print('test date:', min(test['date']), 'to', max(test['date']))<feature_engineering>
def ProcessData(df,features,test_size=0.25):
if test_size == 0:
train_Y = df.pop('scalar_coupling_constant')
 train_type = df.pop('type')
 df.pop('id')
 return df.loc[:,df.columns.map(lambda x: x in features)], train_Y, train_type
train_X, val_X, train_Y, val_Y = train_test_split(df.loc[:,df.columns.map(lambda x: x in features or x=='type')],
df.scalar_coupling_constant,test_size=test_size,random_state=42)
 train_type = train_X.pop('type')
 val_type = val_X.pop('type')
 return train_X, train_Y, train_type, val_X, val_Y, val_type
test['totals.transactionRevenue'] = np.nan<concatenate>
def CalcLMAE(y_true, y_pred, groups, floor=1e-9):
maes =(y_true-y_pred ).abs().groupby(groups ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
all_data = train.append(test, sort=False ).reset_index(drop=True )<sort_values>
def SingleRun(df,features, test_size=0.25, model_fn=XGBRegressor, includeType=False, early_stopping_rounds=None, do_SHAP=False, **kwargs):
data = ProcessData(df,features,test_size)
 if(test_size==0):
train_X,train_Y,train_type = data
else:
train_X,train_Y,train_type,val_X,val_Y,val_type = data
if includeType:
train_X=train_X.join(train_type)
 val_X=val_X.join(val_type)
 enc = LabelEncoder()
train_X.type=enc.fit_transform(train_X.type ).astype(np.uint8)
 val_X.type=enc.transform(val_X.type ).astype(np.uint8)
 model = model_fn(**kwargs)
 t1=time.time()
if early_stopping_rounds is None:
model.fit(train_X,train_Y, verbose=False)
 else:
if test_size==0:
print('ERROR: need test data for early_stopping_rounds')
 return -1,-1,-1
model.fit(train_X,train_Y, early_stopping_rounds=5, eval_set=[(val_X, val_Y)], verbose=False)
 print('best LMAE:',log(model.best_score))
print('best ntree:', model.best_ntree_limit)
 t2=time.time()
print('training time:',t2-t1)
 train_predict = pd.Series(model.predict(train_X),index=train_X.index)
 train_LMAE = CalcLMAE(train_Y,train_predict,train_type)
 print('\ttrain LMAE:',train_LMAE)
 if(test_size==0):
val_LMAE = None
plt.figure(figsize=[6,6])
 sns.scatterplot(x=train_Y, y=train_predict)
 plt.plot(train_Y,train_Y,color='black')
 plt.title(train_type.values[0])
 plt.xlabel('scalar coupling constant')
 plt.ylabel('predicted value')
 plt.show()
if do_SHAP:
samp = train_X.sample(200)
 shap.summary_plot(model.get_booster().predict(xgboost.DMatrix(samp),pred_contribs=True)[:,:-1], samp)
 else:
val_predict = pd.Series(model.predict(val_X),index=val_X.index)
 val_LMAE = CalcLMAE(val_Y,val_predict,val_type)
 print('\tval LMAE:',val_LMAE)
 plt.figure(figsize=[6,6])
 sns.scatterplot(x=val_Y, y=val_predict)
 plt.plot(val_Y,val_Y,color='black')
 plt.title(train_type.values[0])
 plt.xlabel('scalar coupling constant')
 plt.ylabel('predicted value')
 plt.show()
if do_SHAP:
samp = val_X.sample(200)
 shap.summary_plot(model.get_booster().predict(xgboost.DMatrix(samp),pred_contribs=True)[:,:-1], samp)
 gc.collect()
return model, train_LMAE, val_LMAE
null_cnt = train.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )<data_type_conversions>
coupling_type = '3JHN'
train_sample = train[train.type==coupling_type]
features=fc1.copy()
if(coupling_type[0]=='2'):
features.update(fc_2)
 elif(coupling_type[0]=='3'):
features.update(fc_2[:-1]+fc_3)
 model_3JHN,_,_=SingleRun(train_sample,features,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=5,
do_SHAP = True,
max_depth=11, learning_rate=0.1, n_estimators=3000,
verbosity=1,
objective='reg:squarederror', booster='gbtree',tree_method= 'gpu_hist',
n_jobs=4,
gamma=0, min_child_weight=1, max_delta_step=0,
subsample=1,colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5,
random_state=0, seed=None, missing=None, importance_type='gain', eval_metric='mae' )
for col in ['trafficSource.keyword',
'trafficSource.referralPath',
'trafficSource.adContent']:
all_data[col].fillna('unknown', inplace=True)
 all_data['totals.pageviews'].fillna(1, inplace=True)
 all_data['totals.newVisits'].fillna(0, inplace=True)
 all_data['totals.bounces'].fillna(0, inplace=True)
 all_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int)
 all_data['totals.newVisits'] = all_data['totals.newVisits'].astype(int)
 all_data['totals.bounces'] = all_data['totals.bounces'].astype(int)
 all_data['trafficSource.isTrueDirect'].fillna(False, inplace=True )<drop_column>
def RunByType(df,test_size=0.25,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,**kwargs):
model_dict={}
train_LMAE_dict={}
val_LMAE_dict={}
for coupling_type in coupling_types:
print('Now training type:',str(coupling_type))
df_type = df[df['type']==coupling_type]
features=fc1.copy()
if(coupling_type[0]=='2'):
features.update(fc_2)
 elif(coupling_type[0]=='3'):
features.update(fc_2[:-1]+fc_3)
 model,train_LMAE,val_LMAE = SingleRun(df_type,features,test_size,model_fn=model_fn,includeType=False,early_stopping_rounds=early_stopping_rounds,**kwargs)
 model_dict[coupling_type]=model
train_LMAE_dict[coupling_type]=train_LMAE
val_LMAE_dict[coupling_type]=val_LMAE
train_LMAE = sum(train_LMAE_dict.values())/len(coupling_types)
 if test_size>0:
val_LMAE = sum(val_LMAE_dict.values())/len(coupling_types)
 else:
val_LMAE=None
print('total train LMAE:',train_LMAE)
 print('total val LMAE', val_LMAE)
 gc.collect()
return model_dict
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )<sort_values>
def PredictByType(df_X,model_dict,df_Y=None):
predictions = pd.DataFrame()
for coupling_type in coupling_types:
print('predicting type:',str(coupling_type))
model = model_dict[coupling_type]
df_type = df_X[df_X['type']==coupling_type]
features=fc1.copy()
if(coupling_type[0]=='2'):
features.update(fc_2)
 elif(coupling_type[0]=='3'):
features.update(fc_2[:-1]+fc_3)
 predict = model.predict(df_type.loc[:,df_X.columns.map(lambda col: col in features)])
 predict = pd.Series(predict,index=df_type.index)
 predict = pd.DataFrame({'id':df_type.id,'scalar_coupling_constant': predict})
 predict.set_index('id',inplace=True)
 predictions = pd.concat([predictions,predict])
 if df_Y is not None:
predictions.columns = ['predictions']
merged = pd.merge(train,predictions,how='left',left_on='id',right_index=True)
 LMAE = CalcLMAE(df_Y, merged.predictions,merged.type)
 print('LMAE:', LMAE)
 g = sns.FacetGrid(merged,col="type", col_order = coupling_types,sharex=False,sharey=False)
 g.map(sns.scatterplot, "scalar_coupling_constant","predictions")
 del merged
gc.collect()
return predictions
all_data[all_data['fullVisitorId'] == '7813149961404844386'].sort_values(by='visitNumber')[
['date','visitId','visitNumber','totals.hits','totals.pageviews']].head(20 )<count_unique_values>
model_dict=RunByType(train,test_size=0.2,includeType=False,early_stopping_rounds=5,
max_depth=11, learning_rate=0.1, n_estimators=10000,
verbosity=1,
objective='reg:squarederror', booster='gbtree',tree_method= 'gpu_hist',
n_jobs=4,
gamma=0, min_child_weight=1, max_delta_step=0,
subsample=1,colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5,
random_state=0, seed=None, missing=None, importance_type='gain', eval_metric='mae')
 test_predict = PredictByType(test,model_dict)
 test_predict.to_csv('test_predict_TypeModel.csv' )
print('unique customDimensions count:', train['customDimensions'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'customDimensions' )<feature_engineering>
print('FINISHED!' )