Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<data_type_conversions>
print(pd.__version__)
 SEED = 26
LR = 1e-4
print(all_data['totals.pageviews'].value_counts() [:10])
 all_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int )<count_unique_values>
warnings.filterwarnings("ignore")
 warnings.filterwarnings(action="ignore",category=DeprecationWarning)
 warnings.filterwarnings(action="ignore",category=FutureWarning)
 np.random.seed(SEED)
 set_random_seed(SEED)
 def read_pickle(path):
with open(path, "rb")as f:
obj = pickle.load(f)
 return obj
def add_contributions(train, test):
scalar_coupling_contributions = pd.read_csv(
 f"{DATA_PATH}/scalar_coupling_contributions.csv" 
)
 gc.collect()
len_train = len(train)
 df = pd.concat([train, test], axis=0, sort=True)
 del train, test
gc.collect()
df = pd.merge(
 df,
scalar_coupling_contributions,
on=["molecule_name", "atom_index_0", "atom_index_1", "type"],
how="left", 
)
 train = df.iloc[:len_train, :].reset_index(drop=True)
 test = df.iloc[len_train:, :].reset_index(drop=True)
 del df
gc.collect()
return train, test
def select_features_for_type(
 type_,
imp_file=".. /input/cached-champs/lgb_feat_importance_full.csv",
top_n=900,
imp_threshold=100, 
):
df = pd.read_csv(imp_file)
 df = df[ 
(df["type"] == type_)
 &(df["importance"] > 0)
 &(df["importance"] > imp_threshold)
 ]
df = df.sort_values("importance", ascending=False)
 return list(df["feature"].values[:top_n] )
print('unique trafficSource.adContent count:', train['trafficSource.adContent'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.adContent', 10)
 all_data['_adContentGMC'] =(all_data['trafficSource.adContent'] == 'Google Merchandise Collection' ).astype(np.uint8 )<count_unique_values>
print(os.listdir(".. /input"))
FILETRAIN1 = ".. /input/cached-champs/k_cached_processed_train2.pkl"
FILETEST1 = ".. /input/cached-champs/k_cached_processed_test2.pkl"
FILETRAIN = ".. /input/cached-champs/k_cached_processed_train_inv_pvals.pkl"
FILETEST = ".. /input/cached-champs/k_cached_processed_test_inv_pvals.pkl"
epoch_n = 2200
verbose = 1
batch_size = 2048
LINEAR_TOP = 27
LGB_TOP = 17
TARGETS = ['fc', 'sd', 'pso', 'dso']
logger = logging.getLogger('chimps gona win')
 logger.setLevel(logging.DEBUG)
 DATA_PATH = '.. /input/champs-scalar-coupling'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
logger.debug('starting...this is going to be awesome' )
print('unique trafficSource.campaign count:', train['trafficSource.campaign'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.campaign', 10)
 all_data['_withCampaign'] =(all_data['trafficSource.campaign'] != '(not set)' ).astype(np.uint8 )<count_unique_values>
types_dict = {
"1JHC": 0,
"2JHH": 3,
"1JHN": 1,
"2JHN": 4,
"2JHC": 2,
"3JHH": 6,
"3JHC": 5,
"3JHN": 7,
}
atom_features = [
"atom_2",
"atom_3",
"atom_4",
"atom_5",
"atom_6",
"atom_7",
"atom_8",
"atom_9",
]
fc_feats = [
"fc_preds_type",
"fc_preds_akira",
"fc_preds_akira2",
"fc_preds_akira3",
"fc_preds_akira4",
]
types_to_run = ["1JHN", "1JHC", "2JHH", "2JHN", "2JHC", "3JHH", "3JHC", "3JHN"]
all_feats = []
for mol_type in types_to_run:
i = types_dict[mol_type]
linear_feats = select_features_for_type(
 i, top_n=LINEAR_TOP, imp_threshold=0,
imp_file=".. /input/cached-champs/feats_inv_pvals.csv" 
)
 lgb_feats = select_features_for_type(
 i, top_n=LGB_TOP, imp_threshold=0, 
)
 feats = linear_feats + lgb_feats
all_feats.append(feats)
 all_feats = np.unique(list(np.concatenate(all_feats)) + atom_features + fc_feats)
 print(all_feats )
print('unique trafficSource.keyword count:', train['trafficSource.keyword'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.keyword', 10 )<count_unique_values>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 cols = ['molecule_name', 'atom_index_0', 'atom_index_1', 'type']
train_csv = train_csv[cols]
train_csv.head(10)
print('unique trafficSource.medium count:', train['trafficSource.medium'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.medium' )<count_unique_values>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 cols = [col for col in cols if 'scalar_coupling_constant' not in col]
test_csv = test_csv[cols]
gc.collect()
print(train_csv.shape)
 print(test_csv.shape)
 test_csv.head()
print('unique trafficSource.source count:', train['trafficSource.source'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.source', 10)
 all_data['_sourceGpmall'] =(all_data['trafficSource.source'] == 'mall.googleplex.com' ).astype(np.uint8 )<categorify>
train_csv, test_csv = add_contributions(train_csv, test_csv )
_=
all_data['_meanHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('mean')
 all_data['_sumHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('sum')
 for feature in ['totals.hits', 'totals.pageviews']:
info = all_data.groupby('fullVisitorId')[feature].mean()
all_data['_usermean_' + feature] = all_data.fullVisitorId.map(info)
 for feature in ['visitNumber']:
info = all_data.groupby('fullVisitorId')[feature].max()
all_data['_usermax_' + feature] = all_data.fullVisitorId.map(info)
 del info<feature_engineering>
cols = ["molecule_name", "atom_index_0", "atom_index_1"]
train_csv = train_csv.drop(cols, axis=1)
 test_csv = test_csv.drop(cols, axis=1)
 train_csv.head()
all_data['_source.country'] = all_data['trafficSource.source'] + '_' + all_data['geoNetwork.country']
all_data['_campaign.medium'] = all_data['trafficSource.campaign'] + '_' + all_data['trafficSource.medium']
all_data['_browser.category'] = all_data['device.browser'] + '_' + all_data['device.deviceCategory']
all_data['_browser.os'] = all_data['device.browser'] + '_' + all_data['device.operatingSystem']<sort_values>
tr = read_pickle(FILETRAIN)
 tr = tr.fillna(0)
 train_ix = train_csv.index
tr.index = train_ix
train_csv = pd.concat([train_csv, tr], axis=1)
 train_csv.index = train_ix
train_csv = train_csv[[col for col in train_csv.columns if col in list(all_feats)+ TARGETS + ["type"]]]
del tr
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )<data_type_conversions>
te = read_pickle(FILETEST)
 te = te.fillna(0)
 test_ix = test_csv.index
te.index = test_ix
test_csv = pd.concat([test_csv, te], axis=1)
 test_csv.index = test_ix
test_csv = test_csv[[col for col in test_csv.columns if col in list(all_feats)+ ["type"]]]
del te, test_ix, train_ix
gc.collect()
print(train_csv.shape)
 print(test_csv.shape)
 test_csv.head(10 )
all_data.drop(['visitId','visitStartTime'],axis=1,inplace=True)
 for i, t in all_data.loc[:, all_data.columns != 'fullVisitorId'].dtypes.iteritems() :
if t == object:
all_data[i].fillna('unknown', inplace=True)
 all_data[i] = pd.factorize(all_data[i])[0] 
<prepare_x_and_y>
tr = read_pickle(FILETRAIN1)
 tr = tr.fillna(0)[[col for col in tr.columns if col not in train_csv.columns]]
train_ix = train_csv.index
tr.index = train_ix
train_csv = pd.concat([train_csv, tr], axis=1)
 train_csv.index = train_ix
del tr
train_csv = train_csv[list(all_feats)+ TARGETS + ["type"]]
gc.collect()
te = read_pickle(FILETEST1)
 te = te.fillna(0)[[col for col in te.columns if col not in test_csv.columns]]
test_ix = test_csv.index
te.index = test_ix
test_csv = pd.concat([test_csv, te], axis=1)
 test_csv.index = test_ix
del te, test_ix, train_ix
gc.collect()
test_csv = test_csv[list(all_feats)+ ["type"]]
test_csv.head(10)
 print(train_csv.shape)
 print(test_csv.shape)
 gc.collect()
train = all_data[all_data['totals.transactionRevenue'].notnull() ]
test = all_data[all_data['totals.transactionRevenue'].isnull() ].drop(['totals.transactionRevenue'], axis=1 )<prepare_x_and_y>
gc.collect()
print(len(train_csv.columns), len(np.unique(train_csv.columns)) )
train_id = train['fullVisitorId']
test_id = test['fullVisitorId']
Y_train_reg = train.pop('totals.transactionRevenue')
 X_train = train.drop(['fullVisitorId'], axis=1)
 X_test = test.drop(['fullVisitorId'], axis=1)
 print(X_train.shape, X_test.shape )<create_dataframe>
config = tf.ConfigProto(device_count = {'GPU': 1 , 'CPU': 2})
 config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.6
sess = tf.Session(config=config)
 K.set_session(sess )
del all_data, train, test, train_rev
gc.collect()
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<import_modules>
cv_score = []
cv_score_total = 0
retrain = True
start_time = datetime.now()
test_prediction = np.zeros(len(test_csv))
class FeatureTransformer:
def transform(self, dataset, ohe_features=[], continuous_features=[]):
ohe_df = OneHotEncoder().fit_transform(dataset.loc[:, ohe_features] ).toarray()
skews = dataset.loc[:, continuous_features].skew().to_dict()
return np.concatenate(
 [
StandardScaler().fit_transform(dataset.loc[:, continuous_features]),
ohe_df,
],
axis=1, 
)
 for mol_type in types_to_run:
model_name_wrt = "/kaggle/working/molecule_model_%s.hdf5" % mol_type
model_name_rd = ".. /input/nncont-seed-25/molecule_model_%s.hdf5" % mol_type
if mol_type=="3JHC":
linear = 16
lgb = 16
else:
linear = LINEAR_TOP
lgb = LGB_TOP
linear_feats = select_features_for_type(
 types_dict[mol_type], top_n=linear, imp_threshold=0,
imp_file=".. /input/cached-champs/feats_inv_pvals.csv" 
)
 lgb_feats = select_features_for_type(
 types_dict[mol_type], top_n=lgb, imp_threshold=0, 
)
 distance_features = list(np.unique(linear_feats + lgb_feats + fc_feats))
input_features = list(np.unique(distance_features + atom_features))
print(input_features)
 df_train_ = train_csv.loc[train_csv["type"] == mol_type, input_features + TARGETS]
df_test_ = test_csv.loc[test_csv["type"] == mol_type, input_features]
msg = f"training {mol_type}, out of {types_to_run}, train shape: {df_train_.shape}, LR: {LR}"
print(msg)
 logger.debug(msg)
 input_data = FeatureTransformer().transform(
 dataset=pd.concat(
 [df_train_.loc[:, input_features], df_test_.loc[:, input_features]] 
),
ohe_features=atom_features,
continuous_features=distance_features, 
)
 target_data = df_train_.loc[:, TARGETS].values
train_index, cv_index = train_test_split(
 np.arange(len(df_train_)) , random_state=SEED, test_size=0.1 
)
 train_target = target_data[train_index]
cv_target = target_data[cv_index]
train_input = input_data[train_index]
cv_input = input_data[cv_index]
test_input = input_data[len(df_train_):, :]
nn_model = create_nn_model(train_input.shape[1])
 retrain = False
if not retrain:
nn_model = load_model(model_name_rd)
 nn_model.compile(loss="mae", optimizer=Adam(lr=LR))
es = callbacks.EarlyStopping(
 monitor="val_loss",
min_delta=0.0001,
patience=50,
verbose=1,
mode="auto",
restore_best_weights=True, 
)
 rlr = callbacks.ReduceLROnPlateau(
 monitor="val_loss", factor=0.5, patience=30, min_lr=1e-7, mode="auto", verbose=1 
)
 sv_mod = callbacks.ModelCheckpoint(
 model_name_wrt, monitor="val_loss", save_best_only=True, period=1 
)
 history = nn_model.fit(
 train_input,
[train_target],
validation_data=(cv_input, [cv_target]),
callbacks=[es, rlr, sv_mod],
epochs=epoch_n,
batch_size=batch_size,
verbose=0, 
)
 cv_predict = nn_model.predict(cv_input ).sum(axis=1)
 plot_history(history, mol_type)
 accuracy = np.mean(np.abs(cv_target.sum(axis=1)- cv_predict))
print(np.log(accuracy))
cv_score.append(np.log(accuracy))
cv_score_total += np.log(accuracy)
 test_predict = nn_model.predict(test_input ).sum(axis=1)
 test_prediction[test_csv["type"] == mol_type] = test_predict
K.clear_session()
cv_score_total /= len(types_to_run )
from sklearn.model_selection import StratifiedKFold, GroupKFold
from sklearn.metrics import roc_auc_score
import lightgbm as lgb<split>
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')
 def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv("/kaggle/working/nnetCont_sub.csv", index=False)
 submits(test_prediction )
params={'learning_rate': 0.01,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.6,
'feature_fraction': 0.6
}
folds = GroupKFold(n_splits=5)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds.split(X_train, Y_train_reg, groups=train_id)) :
trn_x, trn_y = X_train.iloc[trn_], Y_train_reg.iloc[trn_]
val_x, val_y = X_train.iloc[val_], Y_train_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=3000)
 reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=50, verbose=500)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 sub_preds += reg.predict(X_test, num_iteration=reg.best_iteration_)/ folds.n_splits
pred = sub_preds<save_to_csv>
print('Total training time: ', datetime.now() - start_time)
 i=0
for mol_type in types_to_run:
print(mol_type,": cv score is ",cv_score[i])
 i+=1
print("total cv score is",cv_score_total )
<set_options><EOS> 
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')
 def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv(f"/kaggle/working/nnetCont_sub_{round(cv_score_total, 4)}.csv", index=False)
 submits(test_prediction )