Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<feature_engineering>
EXP_NUMBER = 63
def make_email_feature(df):
for c in ['P_emaildomain', 'R_emaildomain']:
df[c + '_bin'] = df[c].map(emails)
 df[c + '_suffix'] = df[c].map(lambda x: str(x ).split('.')[-1])
 df[c + '_suffix'] = df[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<feature_engineering>
import pandas as pd
import numpy as np
for df in [df_train, df_test]:
df = make_email_feature(df=df )<statistical_test>
import pandas as pd
import numpy as np
def siev_card_feature(df_train, df_test, col):
valid_card = pd.concat([df_train[[col]], df_test[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card_std = valid_card.values.std()
invalid_cards = valid_card[valid_card < 2]
print('{0}{1}
Number of Rare card : '.format(col, '*'*10), len(invalid_cards))
valid_card = valid_card[valid_card >= 2]
valid_card = list(valid_card.index)
 print('Number of not intersection in Train : ', len(df_train[~df_train[col].isin(df_test[col])]))
print('Nmber of intersection in Train : ', len(df_train[df_train[col].isin(df_test[col])]))
df_train[col] = np.where(df_train[col].isin(df_test[col]), df_train[col], np.nan)
 df_test[col] = np.where(df_test[col].isin(df_train[col]), df_test[col], np.nan)
 df_train[col] = np.where(df_train[col].isin(valid_card), df_train[col], np.nan)
 df_test[col] = np.where(df_test[col].isin(valid_card), df_test[col], np.nan )<feature_engineering>
df_train = pd.read_csv('.. /input/champs-scalar-coupling/train.csv')
 df_test = pd.read_csv('.. /input/champs-scalar-coupling/test.csv')
 df_struct = pd.read_csv('.. /input/champs-scalar-coupling/structures.csv' )
for col in ['card1','card2','card3','card4','card5','card6']:
siev_card_feature(df_train, df_test, col )<data_type_conversions>
from sklearn.preprocessing import LabelEncoder
def make_composite_id(new_id, id1, id2):
df_train[new_id] = df_train[id1].astype(str)+'_' + df_train[id2].astype(str)
 df_test[new_id] = df_test[id1].astype(str)+'_' + df_test[id2].astype(str )<remove_duplicates>
type_enc = LabelEncoder()
df_train['type'] = type_enc.fit_transform(df_train['type'])
 df_test['type'] = type_enc.transform(df_test['type'] )
make_composite_id('uid1', 'card1', 'card2')
 make_composite_id('uid2', 'uid1', 'card3')
 make_composite_id('uid2', 'uid2', 'card5')
 make_composite_id('uid3', 'uid2', 'addr1')
 make_composite_id('uid3', 'uid3', 'addr2')
 make_composite_id('uid4', 'uid3', 'P_emaildomain')
 make_composite_id('uid5', 'uid3', 'R_emaildomain')
 make_composite_id('bank_type', 'card3', 'card5' )<set_options>
atom_enc = LabelEncoder()
df_struct['atom'] = atom_enc.fit_transform(df_struct['atom'] )
gc.collect()<categorify>
df_train = df_train.merge(df_struct,
left_on=['molecule_name','atom_index_0'],
right_on=['molecule_name','atom_index'] )
for df in [df_train, df_test]:
df = reduce_mem_usage(df=df )<define_variables>
df_train = df_train.merge(df_struct,
left_on=['molecule_name','atom_index_1'],
right_on=['molecule_name','atom_index'],
suffixes=('_0', '_1'))
def create_new_columns(key, aggs):
return [key + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]<count_unique_values>
n_neighbors = 5
neigh_da_cols = [f'da_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)for typ in range(5)]
neigh_d0_cols = [f'd0_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)for typ in range(5)]
neigh_d1_cols = [f'd1_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)for typ in range(5)]
neigh_cosa_cols = [f'cosine_da_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)
 for typ in range(5)]
neigh_cos0_cols = [f'cosine_d0_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)
 for typ in range(5)]
neigh_cos1_cols = [f'cosine_d1_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)
 for typ in range(5)]
neigh_all_cols = neigh_d0_cols + neigh_d1_cols
neigh_cols = ['id'] + neigh_all_cols
aggs = {}
for col in ['DT_D','DT_W','DT_M']:
aggs[col] = ['nunique', 'mean', 'median', 'var', 'skew']
aggs<merge>
def load_frames(files, columns, parent_folder='.. /input/dist-feats'):
df_neigh = pd.DataFrame()
for name in files:
print(f'Loading {name}...')
 dfn = pd.read_csv(f'{parent_folder}/{name}', usecols=columns)
 df_neigh = pd.concat([df_neigh, dfn])
 del(dfn)
 return df_neigh
df_temp = pd.concat([df_train, df_test])
 for key in tqdm_notebook(['card3', 'card5', 'bank_type']):
new_columns = create_new_columns(key, aggs)
 df_grouped = df_temp.groupby(key ).agg(aggs)
 df_grouped.columns = new_columns
df_grouped.reset_index(drop=False,inplace=True)
 df_train = df_train.merge(df_grouped,on=key,how='left')
 df_test = df_test.merge(df_grouped,on=key,how='left' )<define_variables>
df_train_neigh_files = [
'top-5_atoms-['sep2']_count-1164536_44c3ddd5-0dd4-42b7-acac-d0a0e7bd444e.csv',
'top-5_atoms-['sep2']_count-1164536_7e41a7b8-d1ed-4d11-be5e-5166d347a8a2.csv',
'top-5_atoms-['sep2']_count-1164536_e2e300a0-55c4-453a-b40d-b62e4cf7b336.csv',
'top-5_atoms-['sep2']_count-1164539_42ba6898-a223-4a78-8157-b1f015c09806.csv',
]
df_neigh = load_frames(df_train_neigh_files, neigh_cols )
aggs = {}
for col in ['TransactionAmt', 'D15']:
aggs[col] = ['nunique', 'mean', 'median', 'var', 'skew', 'min', 'max']
aggs<merge>
df_train = df_train.merge(df_neigh, on='id' )
for key in tqdm_notebook(['card1','card2','card3','card5','uid1','uid2','uid3','uid4','uid5','bank_type']):
new_columns = create_new_columns(key, aggs)
 df_grouped = df_temp.groupby(key ).agg(aggs)
 df_grouped.columns = new_columns
df_grouped.reset_index(drop=False,inplace=True)
 df_train = df_train.merge(df_grouped,on=key,how='left')
 df_test = df_test.merge(df_grouped,on=key,how='left' )<feature_engineering>
n_neighbors = 5
neigh_dm_cols_2 = [f'dm_med2_neighbor{i}' for i in range(1, n_neighbors+1)]
neigh_d0_cols_2 = [f'd0_med2_neighbor{i}' for i in range(1, n_neighbors+1)]
neigh_d1_cols_2 = [f'd1_med2_neighbor{i}' for i in range(1, n_neighbors+1)]
neigh_cos0_cols_2 = [f'cos0_med2_neighbor{i}' for i in range(1, n_neighbors+1)]
neigh_all_cols_2 = neigh_d0_cols_2 + neigh_d1_cols_2
neigh_cols_2 = ['id'] + neigh_all_cols_2
aggs = {}
for col in ['TransactionAmt', 'D15']:
aggs[col] = ['mean', 'median', 'min', 'max']
for key in tqdm_notebook(['card1','card2','card3','card5','uid1','uid2','uid3','uid4','uid5','bank_type']):
columns = create_new_columns(key, aggs)
 for i in columns:
diff_column = 'diff_' + i
ratio_column = 'ratio_' + i
df_train[diff_column] = df_train['TransactionAmt'] - df_train[i]
df_test[diff_column] = df_test['TransactionAmt'] - df_test[i]
df_train[ratio_column] = df_train['TransactionAmt'] / df_train[i]
df_test[ratio_column] = df_test['TransactionAmt'] / df_test[i]<set_options>
df_train_neigh_files = [
'top-5_atoms-['med2']_count-1164536_0a918683-2fe3-43a1-b239-c900712c1433.csv',
'top-5_atoms-['med2']_count-1164536_b845e873-855d-424b-a7f3-50d89824b0ea.csv',
'top-5_atoms-['med2']_count-1164536_df446883-d0af-486c-877b-f8d834c0b00f.csv',
'top-5_atoms-['med2']_count-1164539_de6e2745-1a72-490c-ac29-69ad07c56910.csv',
]
df_neigh = load_frames(df_train_neigh_files, neigh_cols_2 )
del df_temp
del df_grouped
gc.collect()<count_missing_values>
df_train = df_train.merge(df_neigh, on='id' )
df_train['nulls1'] = df_train.isna().sum(axis=1)
 df_test['nulls1'] = df_test.isna().sum(axis=1 )<define_variables>
df_train.set_index('id', inplace=True)
 df_train.sort_index(inplace=True )
categorical_features = list()
for i in df_train.columns:
if(df_train[i].dtype == object):
categorical_features.append(i)
 print(categorical_features )<categorify>
df_train['atom_0'].value_counts() , df_train['atom_1'].value_counts()
def label_encoding(df_train, df_test, feature):
le = preprocessing.LabelEncoder()
le.fit(list(df_train[feature].astype(str ).values)+ list(df_test[feature].astype(str ).values))
df_train[feature] = le.transform(list(df_train[feature].astype(str ).values))
df_test[feature] = le.transform(list(df_test[feature].astype(str ).values))<categorify>
df_train = df_train.drop(['atom_0'], axis=1 )
for feature in tqdm_notebook(categorical_features):
label_encoding(df_train, df_test, feature )<categorify>
df_test = df_test.merge(df_struct,
left_on=['molecule_name','atom_index_0'],
right_on=['molecule_name','atom_index'],
sort=False )
def frequency_encoding(df_train, df_test, feature, self_encoding=False):
df_temp = pd.concat([df_train[[feature]], df_test[[feature]]])
 fq_encode = df_temp[feature].value_counts(dropna=False ).to_dict()
if self_encoding:
df_train[feature] = df_train[feature].map(fq_encode)
 df_test[feature] = df_test[feature].map(fq_encode)
 else:
df_train[feature+'_fq_enc'] = df_train[feature].map(fq_encode)
 df_test[feature+'_fq_enc'] = df_test[feature].map(fq_encode )<categorify>
df_test = df_test.merge(df_struct,
left_on=['molecule_name','atom_index_1'],
right_on=['molecule_name','atom_index'],
suffixes=('_0', '_1'),
sort=False )
for feature in tqdm_notebook(categorical_features):
frequency_encoding(df_train, df_test, feature, self_encoding=False )<categorify>
df_test_neigh_files = [
'top-5_atoms-['sep2']_count-626385_7623bd50-41b8-4c44-b40a-e37837700460.csv',
'top-5_atoms-['sep2']_count-626385_878ae0ca-67c7-491e-b580-fbdc384c2477.csv',
'top-5_atoms-['sep2']_count-626385_e4f75a2d-302c-476b-b60c-c5624925ef01.csv',
'top-5_atoms-['sep2']_count-626387_4792052c-d539-4e62-8e37-c4ecf81ef669.csv',
]
df_neigh = load_frames(df_test_neigh_files, neigh_cols )
for feature in tqdm_notebook(['DT_M','DT_W','DT_D']):
frequency_encoding(df_train, df_test, feature, self_encoding=False )<categorify>
df_test = df_test.merge(df_neigh, on='id' )
for feature in tqdm_notebook(['card1','card2','card3','card5','uid1','uid2','uid3','uid4','uid5']):
frequency_encoding(df_train, df_test, feature, self_encoding=False )<categorify>
df_test_neigh_files = [
'top-5_atoms-['med2']_count-626385_0072225a-03f6-42bf-b1ed-b81be41c6d14.csv',
'top-5_atoms-['med2']_count-626385_7ed7d305-6c6a-4d3c-a6e8-f8e274fd461f.csv',
'top-5_atoms-['med2']_count-626385_b4f9dd09-7c61-42a7-9534-85a31496d1a5.csv',
'top-5_atoms-['med2']_count-626387_a9d566a5-1248-4341-abec-b71a87bbe21d.csv',
]
df_neigh = load_frames(df_test_neigh_files, neigh_cols_2 )
for feature in tqdm_notebook(['general_vendor','general_platform','general_type']):
frequency_encoding(df_train, df_test, feature, self_encoding=False )<categorify>
df_test = df_test.merge(df_neigh, on='id' )
for df in [df_train, df_test]:
df = reduce_mem_usage(df=df )<set_options>
df_test.set_index('id', inplace=True)
 df_test.sort_index(inplace=True )
gc.collect()<load_pretrained>
df_test['atom_0'].value_counts() , df_test['atom_1'].value_counts()
df_train.to_pickle('df_train.pkl')
 df_test.to_pickle('df_test.pkl' )<set_options>
df_test = df_test.drop(['atom_0'], axis=1 )
sns.set_style('darkgrid')
 warnings.filterwarnings("ignore")
<load_from_csv>
del(df_struct)
 del(df_neigh )
train_trains = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv', index_col = 'TransactionID')
 train_id = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv', index_col = 'TransactionID')
 test_trains = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv', index_col = 'TransactionID')
 test_id = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv', index_col = 'TransactionID' )<categorify>
def calc_polar(df, suffix_0='_0', suffix_1='_1', angles=True):

 dx = df['x'+suffix_0] - df['x'+suffix_1]
dy = df['y'+suffix_0] - df['y'+suffix_1]
dz = df['z'+suffix_0] - df['z'+suffix_1]
if angles:
dx2 = np.power(dx, 2)
 dy2 = np.power(dy, 2)
 dz2 = np.power(dz, 2)
 d = np.sqrt(dx2 + dy2 + dz2)
 az = np.arctan2(dy, dx)
 el = np.arctan2(dz, np.sqrt(dx2 + dy2))
return d, az, el
else:
return np.sqrt(dx**2 + dy**2 + dz**2 )
train_trains = reduce_mem_usage(train_trains)
 train_id = reduce_mem_usage(train_id)
 test_trains = reduce_mem_usage(test_trains)
 test_id = reduce_mem_usage(test_id )<merge>
def feat_eng(df):
d, _, _ = calc_polar(df)
 df_dummy = pd.get_dummies(df[['atom_1']], columns=['atom_1'])
 atom_1_cols = [x for x in df_dummy.columns if 'atom_1' in x]
d_cols = [f'd_{x[7:]}' for x in atom_1_cols]
df[d_cols] = df_dummy[atom_1_cols].multiply(d, axis='index' )
train = pd.merge(train_trains, train_id, on ='TransactionID', how = 'left')
 test = pd.merge(test_trains, test_id, on = 'TransactionID', how = 'left')
 train = train.reset_index()
test = test.reset_index()
print(train.shape)
 print(test.shape )<drop_column>
import matplotlib.pyplot as plt
del train_id, train_trains, test_id, test_trains
gc.collect()<count_values>
del(corr )
print('This might be an imbalanced class problem from what we can see')
 train['isFraud'].value_counts(normalize = True)*100<count_missing_values>
target = 'scalar_coupling_constant'
print('THERE ARE {0} MISSING VALUES IN TRAIN'.format(train.isnull().any().sum()))<count_values>
d_cols = [x for x in df_train.columns if x.startswith('d_')and len(x)== 3]
feat_cols = d_cols + neigh_all_cols + neigh_all_cols_2
train['isFraud'].value_counts(normalize = True, dropna = False ).values<count_values>
y_train = df_train[target].values
for cols in train.columns[2:]:
if train[cols].value_counts(normalize = True, dropna = False ).values[0]> 0.9:
print(train[cols].value_counts(normalize = True, dropna = False)*100)
 print('-'*90)
 print('-'*90 )<categorify>
del(df_train)
 del(df_test )
def label_collector(string):
label = string.split('.')[0]
return label
temp = train['P_emaildomain'].astype(str)
 train['label_encode'] = temp.apply(label_collector )<drop_column>
def split(X, y, types, random_state=1, p_value=0.9, test_size=0.2, verbose=True):
X_train, X_test, y_train, y_test, train_types, test_types = None, None, None, None, None, None
while True:
X_train, X_test, y_train, y_test, train_types, test_types = train_test_split(
 X, y, types,
test_size=test_size,
stratify=types,
random_state=random_state 
)
 if p_value is None:
return X_train, X_test, y_train, y_test, train_types, test_types
st1 = ks_2samp(y, y_train)
 st2 = ks_2samp(y, y_test)
 st3 = ks_2samp(y_train, y_test)
 if verbose:
print('RS =', random_state)
 print(st1)
 print(st2)
 print(st3)
 if st1.pvalue > p_value and st2.pvalue > p_value and st3.pvalue > p_value:
return X_train, X_test, y_train, y_test, train_types, test_types
del(X_train)
 del(X_test)
 del(y_train)
 del(y_test)
 del(train_types)
 del(test_types)
 random_state += 1
train = train.drop('label_encode', axis = 1 )<count_missing_values>
X_train_part, X_valid, y_train_part, y_valid, train_part_types, valid_types = split(X_train, y_train, train_types,
random_state=42, p_value=0.0 )
cols_drop_train = [cols for cols in train.columns if train[cols].isnull().sum() / train.shape[0] > 0.9]
cols_drop_test = [cols for cols in test.columns if test[cols].isnull().sum() / test.shape[0]> 0.9]
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
big_top_value_cols_test = [col for col in test.columns if test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
drop_cols = list(set(cols_drop_train + cols_drop_test + big_top_value_cols + big_top_value_cols_test))
drop_cols.remove('isFraud' )<drop_column>
X_train_small, _, y_train_small, _, train_small_types, _ = split(X_train, y_train, train_types, test_size=0.90 )
train.drop(drop_cols, axis = 1, inplace = True)
 test.drop(drop_cols, axis = 1, inplace = True )<drop_column>
feat_cols_selected_indices = [idx for idx, name in enumerate(feat_cols)if name in feat_cols_selected]
X_train = X_train[:, feat_cols_selected_indices]
X_train_part = X_train_part[:, feat_cols_selected_indices]
X_valid = X_valid[:, feat_cols_selected_indices]
X_train_small = X_train_small[:, feat_cols_selected_indices]
X_test = X_test[:, feat_cols_selected_indices]
del cols_drop_test, cols_drop_train, big_top_value_cols, big_top_value_cols_test, drop_cols<string_transform>
from sklearn.ensemble import ExtraTreesRegressor
train[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = train['P_emaildomain'].str.split('.', expand=True)
 train[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = train['R_emaildomain'].str.split('.', expand=True)
 test[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = test['P_emaildomain'].str.split('.', expand=True)
 test[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = test['R_emaildomain'].str.split('.', expand=True )<categorify>
reg = ExtraTreesRegressor(n_estimators=10, n_jobs=-1, random_state=42, verbose=2 )
def labelencode(train,test):
for col in train.drop(['TransactionID','isFraud','TransactionDT'],axis = 1 ).columns:
if train[col].dtype == 'O' or test[col].dtype == 'O':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
return train,test<categorify>
%%time
reg.fit(X_train_part, y_train_part )
train, test = labelencode(train, test )<prepare_x_and_y>
y_pred = reg.predict(X_valid )
y_test = train['isFraud']<drop_column>
def metric(y, y_hat, types, type_filter=None, verbose=True):
res = 0
uniq_types = np.unique(types)
 if verbose:
print('typ| cnt(%)| sum | log')
 cnt_total = types.shape[0]
for typ in uniq_types:
if type_filter is not None and type_filter != typ:
continue
idx = np.where(types == typ)[0]
cnt = idx.shape[0]
res_typ = np.sum(np.abs(y[idx] - y_hat[idx])) / cnt
assert cnt == y[idx].shape[0] == y_hat[idx].shape[0], 'inconsistent idx'
if verbose:
print(f'{typ:3}|{cnt:8}({1+cnt*100//cnt_total:02d})|{res_typ:12.9}|{np.log(res_typ)}')
 res += np.log(res_typ)
 uniq_types_cnt = uniq_types.shape[0] if type_filter is None else 1
res /= uniq_types_cnt
if verbose:
print(f'Result: {res} for {uniq_types_cnt} types')
 return res
cols_drops = ['TransactionID','isFraud','TransactionDT']
train = train.drop(cols_drops, axis = 1 )<drop_column>
metric(y_valid, y_pred, valid_types )
test = test.drop(['TransactionID','TransactionDT'], axis = 1 )<categorify>
from skopt.space import Real, Integer, Categorical
train = train.fillna(-999)
 test = test.fillna(-999 )<split>
space = [
Integer(10, 100, name='n_estimators'),
Real(0.1, 1.0, prior='uniform', name='max_features'),
]
train_m, val_m_train, val1, val2 = train_test_split(train,y_test, test_size = 0.3, random_state = 10, stratify = y_test)
 train_m_index = train_m.index
val_m_index = val_m_train.index
val1_index = val1.index
val2_index = val2.index<compute_train_metric>
n_estimators_by_type = {
0: 32, 1: 160, 2: 28, 3: 100, 4: 160, 5: 28, 6: 32, 7: 160
}
def objective(num_leaves,min_child_weight,feature_fraction,bagging_fraction,
max_depth,learning_rate,reg_alpha,reg_lambda,min_data_in_leaf):
global train_m
global val_m
global y_test
global train_m_index
global val_m_index
global val1,val2, val1_index, val2_index
num_leaves = int(num_leaves)
 max_depth = int(max_depth)
 min_data_in_leaf = int(min_data_in_leaf)
 assert type(num_leaves)== int
assert type(min_data_in_leaf)== int
assert type(max_depth)== int
params = {'num_leaves': num_leaves,
'min_child_weight': min_child_weight,
'feature_fraction': feature_fraction,
'bagging_fraction': bagging_fraction,
'min_data_in_leaf': min_data_in_leaf,
'objective': 'binary',
'max_depth': max_depth,
'learning_rate': learning_rate,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': reg_alpha,
'reg_lambda': reg_lambda,
'random_state':42,
}
oof = np.zeros(len(train_m))
early_stopping_rounds = 50
xgtrain = lgb.Dataset(train_m, label=val1[val1_index])
 xgvalid = lgb.Dataset(val_m_train, label=val2[val2_index])
 num_boost_round = 200
model_lgb = lgb.train(params, xgtrain , valid_sets = [xgtrain, xgvalid], num_boost_round = num_boost_round,
early_stopping_rounds = early_stopping_rounds, verbose_eval = 0)
 score = roc_auc_score(val2, model_lgb.predict(val_m_train))
return score<define_search_space>
def get_estimator(estimator_model='rf', n_estimators=10, verbose=False):
if estimator_model == 'rf':
cls = RandomForestRegressor
elif estimator_model == 'et':
cls = ExtraTreesRegressor
elif estimator_model == 'lgbm':
cls = LGBMRegressor
else:
return None
print('Selected estimator class:', cls)
 return cls(n_estimators=n_estimators, n_jobs=-1, random_state=42, verbose=verbose )
bound_lgb = {'num_leaves':(70,600),
'min_child_weight':(0.001, 0.07),
'feature_fraction':(0.1,0.9),
'bagging_fraction':(0.1,0.9),
'max_depth':(-1,50),
'learning_rate':(0.2,0.9),
'reg_alpha':(0.3,0.9),
'reg_lambda':(0.3,0.9),
'min_data_in_leaf':(50,300)
 }<choose_model_class>
def skopt_hyper(space, n_estimators_by_type,
X_train, y_train, train_types,
X_valid, y_valid, valid_types,
estimator_model='rf', n_calls=30,
random_state=42, verbose=True):
reg = get_estimator(estimator_model, verbose=0)
 if reg is None:
return None
res_opts = {}
for typ in tqdm_notebook(np.unique(train_types)) :
@use_named_args(space)
 def objective(**params):
reg.set_params(**params)
 mask = train_types == typ
reg.fit(X_train[mask], y_train[mask])
 y_valid_pred = reg.predict(X_valid)
 score = metric(y_valid, y_valid_pred, valid_types, typ, verbose=verbose)
 return score
if n_estimators_by_type is not None:
space[0] = Integer(10, n_estimators_by_type[typ], name='n_estimators')
 if verbose:
print(f'--- Type {typ}, space {space} ---')
 res_opt = gp_minimize(objective, space, n_calls=n_calls,
verbose=True, random_state=random_state)
 res_opts[typ] = res_opt
if verbose:
print(f'--- optimal params: {res_opt.x} ---')
 for typ in np.unique(train_types):
if verbose:
print('Type', typ)
 res_opt = res_opts[typ]
plot_convergence(res_opt)
 plot_evaluations(res_opt)
 plot_objective(res_opt)
 plt.show()
return {k: v.x for k, v in res_opts.items() }
LGB_BO = BayesianOptimization(objective, bound_lgb, random_state=42 )<find_best_params>
%time
res_opts = {
0: [32, 0.9356368713812235],
1: [160, 0.9226985331078089],
2: [28, 0.978079768528465],
3: [100, 1.0],
4: [160, 1.0],
5: [28, 0.9688476690205474],
6: [32, 1.0],
7: [160, 0.9976351777608269]
}
init_points = 10
n_iter = 15
print('-' * 130)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter,acq='ucb', xi=0.0, alpha=1e-5 )<init_hyperparams>
def train_and_predict_sep(X_train, y_train, X_train_part, y_train_part, X_valid, y_valid,
train_types, train_part_types, valid_types,
X_test=None, test_types=None,
estimator_model='rf', cnt_estimators=None, res_opts=None, params=None):
assert X_train.shape[0] == y_train.shape[0] == train_types.shape[0], 'Inconsistent train - X and y'
assert X_train_part.shape[0] == y_train_part.shape[0] == train_part_types.shape[0], 'Inconsistent train_part - X and y'
assert X_valid.shape[0] == y_valid.shape[0] == valid_types.shape[0], 'Inconsistent valid - X and y'
y_pred_valid = np.empty(X_valid.shape[0])
 y_pred_test = None
if X_test is not None and test_types is not None:
assert X_test.shape[0] == test_types.shape[0], 'Inconsistent train - X and y'
y_pred_test = np.empty(X_test.shape[0])
 typ_cnt = np.unique(train_types ).shape[0]
if cnt_estimators is None:
cnt_estimators = [10] * typ_cnt
assert len(cnt_estimators)== typ_cnt, 'Inconsistent estimators count'
for n_estimators, typ in tqdm_notebook(zip(cnt_estimators, range(typ_cnt)) , total=len(cnt_estimators)) :
print('+++++ Fitting model for typ', typ)
 mask = train_part_types == typ
reg = get_estimator(estimator_model, cnt_estimators, verbose=2)
 if reg is None:
return None
if params is not None:
reg.set_params(**params[typ])
 elif res_opts is not None:
res_opt = res_opts[typ]
reg_params = {
'n_estimators': res_opt[0],
'max_features': res_opt[1],
}
reg.set_params(**reg_params)
 print(reg)
 reg.fit(X_train_part[mask], y_train_part[mask])
 print('+++++ Predicting with model for typ', typ)
 mask = valid_types == typ
y_pred_valid[mask] = reg.predict(X_valid[mask])
 metric(y_valid, y_pred_valid, valid_types, typ)
 if X_test is not None and test_types is not None:
print('+++++ Fitting model for typ', typ)
 mask = train_types == typ
reg.fit(X_train[mask], y_train[mask])
 print('+++++ Predicting with model for typ', typ)
 mask = test_types == typ
y_pred_test[mask] = reg.predict(X_test[mask])
 print('+++++ Overall validation report')
 metric(y_valid, y_pred_valid, valid_types)
 return y_pred_test, y_pred_valid
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
}<create_dataframe>
cnt_estimators = None
d_train = lgb.Dataset(train, label=y_test )<train_model>
%%time
y_pred, _ = train_and_predict_sep(X_train, y_train, X_train_part, y_train_part, X_valid, y_valid,
train_types, train_part_types, valid_types,
X_test=X_test, test_types=test_types,
estimator_model='et', cnt_estimators=cnt_estimators, res_opts=res_opts )
clf = lgb.train(params, d_train, verbose_eval=False, num_boost_round = 1000 )<predict_on_test>
df_subm = pd.read_csv('.. /input/champs-scalar-coupling/sample_submission.csv', index_col='id' )
predict = clf.predict(test )<load_from_csv>
df_subm[target] = y_pred
submitss = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv' )<save_to_csv>
fn_csv = f'./{EXP_NUMBER}.subm.csv'
<load_pretrained><EOS> 
df_subm.to_csv(fn_csv )