Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<load_from_csv>
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore' )
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )<find_best_params>
%%time
folder_path = '.. /input/'
print('Loading data...')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_identity!')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_transaction!')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_identity!')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_transaction!')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 print('\tSuccessfully loaded sample_submission!')
 print('Data was successfully loaded!
' )
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
N_FOLDS = 5
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score<save_to_csv>
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )<load_from_csv>
train_identity = id_split(train_identity)
 test_identity = id_split(test_identity )
sub1 = pd.read_csv('.. /input/blend/submission1236.csv')
 sub2 = pd.read_csv('.. /input/statstack/stack_median.csv')
 submission = pd.read_csv('.. /input/champs-scalar-coupling/sample_submission.csv' )<save_to_csv>
print('Merging data...')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')
 gc.collect()
submission['scalar_coupling_constant'] =(0.4*sub1['scalar_coupling_constant'] + 0.6*sub2['scalar_coupling_constant'])
 submission.to_csv('blend_of_blends.csv', index=False )<set_options>
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore" )<load_from_csv>
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT' )
file_folder = '.. /input/champs-scalar-coupling'
train = pd.read_csv(f'{file_folder}/train.csv')
 test = pd.read_csv(f'{file_folder}/test.csv')
 sub = pd.read_csv(f'{file_folder}/sample_submission.csv')
 structures = pd.read_csv(f'{file_folder}/structures.csv' )<load_from_csv>
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
path = '.. /input/parallelization-of-coulomb-yukawa-interaction'
structures_yukawa = pd.read_csv(f'{path}/structures_yukawa.csv')
 structures = pd.concat([structures, structures_yukawa], axis=1)
 del structures_yukawa<prepare_x_and_y>
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )
y = train['scalar_coupling_constant']
train = train.drop(columns = ['scalar_coupling_constant'] )<merge>
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
def map_atom_info(df_1, df_2, atom_idx):
df = pd.merge(df_1, df_2, how = 'left',
left_on = ['molecule_name', f'atom_index_{atom_idx}'],
right_on = ['molecule_name', 'atom_index'])
 df = df.drop('atom_index', axis=1)
 return df
def find_dist(df):
df_p_0 = df[['x_0', 'y_0', 'z_0']].values
df_p_1 = df[['x_1', 'y_1', 'z_1']].values
df['dist'] = np.linalg.norm(df_p_0 - df_p_1, axis=1)
 df['dist_inv2'] = 1/df['dist']**2
df['dist_x'] =(df['x_0'] - df['x_1'])** 2
df['dist_y'] =(df['y_0'] - df['y_1'])** 2
df['dist_z'] =(df['z_0'] - df['z_1'])** 2
return df
def find_closest_atom(df):
df_temp = df.loc[:,["molecule_name",
"atom_index_0","atom_index_1",
"dist","x_0","y_0","z_0","x_1","y_1","z_1"]].copy()
df_temp_ = df_temp.copy()
df_temp_ = df_temp_.rename(columns={'atom_index_0': 'atom_index_1',
'atom_index_1': 'atom_index_0',
'x_0': 'x_1',
'y_0': 'y_1',
'z_0': 'z_1',
'x_1': 'x_0',
'y_1': 'y_0',
'z_1': 'z_0'})
 df_temp_all = pd.concat(( df_temp,df_temp_),axis=0)
 df_temp_all["min_distance"]=df_temp_all.groupby(['molecule_name',
'atom_index_0'])['dist'].transform('min')
 df_temp_all["max_distance"]=df_temp_all.groupby(['molecule_name',
'atom_index_0'])['dist'].transform('max')
 df_temp = df_temp_all[df_temp_all["min_distance"]==df_temp_all["dist"]].copy()
df_temp = df_temp.drop(['x_0','y_0','z_0','min_distance'], axis=1)
 df_temp = df_temp.rename(columns={'atom_index_0': 'atom_index',
'atom_index_1': 'atom_index_closest',
'dist': 'distance_closest',
'x_1': 'x_closest',
'y_1': 'y_closest',
'z_1': 'z_closest'})
 df_temp = df_temp.drop_duplicates(subset=['molecule_name', 'atom_index'])
 for atom_idx in [0,1]:
df = map_atom_info(df,df_temp, atom_idx)
 df = df.rename(columns={'atom_index_closest': f'atom_index_closest_{atom_idx}',
'distance_closest': f'distance_closest_{atom_idx}',
'x_closest': f'x_closest_{atom_idx}',
'y_closest': f'y_closest_{atom_idx}',
'z_closest': f'z_closest_{atom_idx}'})
 df_temp= df_temp_all[df_temp_all["max_distance"]==df_temp_all["dist"]].copy()
df_temp = df_temp.drop(['x_0','y_0','z_0','max_distance'], axis=1)
 df_temp= df_temp.rename(columns={'atom_index_0': 'atom_index',
'atom_index_1': 'atom_index_farthest',
'dist': 'distance_farthest',
'x_1': 'x_farthest',
'y_1': 'y_farthest',
'z_1': 'z_farthest'})
 df_temp = df_temp.drop_duplicates(subset=['molecule_name', 'atom_index'])
 for atom_idx in [0,1]:
df = map_atom_info(df,df_temp, atom_idx)
 df = df.rename(columns={'atom_index_farthest': f'atom_index_farthest_{atom_idx}',
'distance_farthest': f'distance_farthest_{atom_idx}',
'x_farthest': f'x_farthest_{atom_idx}',
'y_farthest': f'y_farthest_{atom_idx}',
'z_farthest': f'z_farthest_{atom_idx}'})
 return df
def add_cos_features(df):
df["distance_center0"] = np.sqrt(( df['x_0']-df['c_x'])**2 \
+(df['y_0']-df['c_y'])**2 \
+(df['z_0']-df['c_z'])**2)
 df["distance_center1"] = np.sqrt(( df['x_1']-df['c_x'])**2 \
+(df['y_1']-df['c_y'])**2 \
+(df['z_1']-df['c_z'])**2)
 df['distance_c0'] = np.sqrt(( df['x_0']-df['x_closest_0'])**2 + \ 
(df['y_0']-df['y_closest_0'])**2 + \ 
(df['z_0']-df['z_closest_0'])**2)
 df['distance_c1'] = np.sqrt(( df['x_1']-df['x_closest_1'])**2 + \ 
(df['y_1']-df['y_closest_1'])**2 + \ 
(df['z_1']-df['z_closest_1'])**2)
 df["distance_f0"] = np.sqrt(( df['x_0']-df['x_farthest_0'])**2 + \ 
(df['y_0']-df['y_farthest_0'])**2 + \ 
(df['z_0']-df['z_farthest_0'])**2)
 df["distance_f1"] = np.sqrt(( df['x_1']-df['x_farthest_1'])**2 + \ 
(df['y_1']-df['y_farthest_1'])**2 + \ 
(df['z_1']-df['z_farthest_1'])**2)
 vec_center0_x =(df['x_0']-df['c_x'])/(df["distance_center0"]+1e-10)
 vec_center0_y =(df['y_0']-df['c_y'])/(df["distance_center0"]+1e-10)
 vec_center0_z =(df['z_0']-df['c_z'])/(df["distance_center0"]+1e-10)
 vec_center1_x =(df['x_1']-df['c_x'])/(df["distance_center1"]+1e-10)
 vec_center1_y =(df['y_1']-df['c_y'])/(df["distance_center1"]+1e-10)
 vec_center1_z =(df['z_1']-df['c_z'])/(df["distance_center1"]+1e-10)
 vec_c0_x =(df['x_0']-df['x_closest_0'])/(df["distance_c0"]+1e-10)
 vec_c0_y =(df['y_0']-df['y_closest_0'])/(df["distance_c0"]+1e-10)
 vec_c0_z =(df['z_0']-df['z_closest_0'])/(df["distance_c0"]+1e-10)
 vec_c1_x =(df['x_1']-df['x_closest_1'])/(df["distance_c1"]+1e-10)
 vec_c1_y =(df['y_1']-df['y_closest_1'])/(df["distance_c1"]+1e-10)
 vec_c1_z =(df['z_1']-df['z_closest_1'])/(df["distance_c1"]+1e-10)
 vec_f0_x =(df['x_0']-df['x_farthest_0'])/(df["distance_f0"]+1e-10)
 vec_f0_y =(df['y_0']-df['y_farthest_0'])/(df["distance_f0"]+1e-10)
 vec_f0_z =(df['z_0']-df['z_farthest_0'])/(df["distance_f0"]+1e-10)
 vec_f1_x =(df['x_1']-df['x_farthest_1'])/(df["distance_f1"]+1e-10)
 vec_f1_y =(df['y_1']-df['y_farthest_1'])/(df["distance_f1"]+1e-10)
 vec_f1_z =(df['z_1']-df['z_farthest_1'])/(df["distance_f1"]+1e-10)
 vec_x =(df['x_1']-df['x_0'])/df['dist']
vec_y =(df['y_1']-df['y_0'])/df['dist']
vec_z =(df['z_1']-df['z_0'])/df['dist']
df["cos_c0_c1"] = vec_c0_x*vec_c1_x + vec_c0_y*vec_c1_y + vec_c0_z*vec_c1_z
df["cos_f0_f1"] = vec_f0_x*vec_f1_x + vec_f0_y*vec_f1_y + vec_f0_z*vec_f1_z
df["cos_c0_f0"] = vec_c0_x*vec_f0_x + vec_c0_y*vec_f0_y + vec_c0_z*vec_f0_z
df["cos_c1_f1"] = vec_c1_x*vec_f1_x + vec_c1_y*vec_f1_y + vec_c1_z*vec_f1_z
df["cos_center0_center1"] = vec_center0_x*vec_center1_x \
+ vec_center0_y*vec_center1_y \
+ vec_center0_z*vec_center1_z
df["cos_c0"] = vec_c0_x*vec_x + vec_c0_y*vec_y + vec_c0_z*vec_z
df["cos_c1"] = vec_c1_x*vec_x + vec_c1_y*vec_y + vec_c1_z*vec_z
df["cos_f0"] = vec_f0_x*vec_x + vec_f0_y*vec_y + vec_f0_z*vec_z
df["cos_f1"] = vec_f1_x*vec_x + vec_f1_y*vec_y + vec_f1_z*vec_z
df["cos_center0"] = vec_center0_x*vec_x + vec_center0_y*vec_y + vec_center0_z*vec_z
df["cos_center1"] = vec_center1_x*vec_x + vec_center1_y*vec_y + vec_center1_z*vec_z
return df
def add_dist_features(df):
df[f'molecule_atom_index_0_dist_mean'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('mean')
 df[f'molecule_atom_index_0_dist_mean_diff'] = df[f'molecule_atom_index_0_dist_mean'] - df['dist']
df[f'molecule_atom_index_0_dist_min'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('min')
 df[f'molecule_atom_index_0_dist_min_diff'] = df[f'molecule_atom_index_0_dist_min'] - df['dist']
df[f'molecule_atom_index_0_dist_std'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('std')
 df[f'molecule_atom_index_1_dist_mean'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('mean')
 df[f'molecule_atom_index_1_dist_mean_diff'] = df[f'molecule_atom_index_1_dist_mean'] - df['dist']
df[f'molecule_atom_index_1_dist_min'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('min')
 df[f'molecule_atom_index_1_dist_min_diff'] = df[f'molecule_atom_index_1_dist_min'] - df['dist']
df[f'molecule_atom_index_1_dist_std'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('std')
 df[f'molecule_type_dist_mean'] = df.groupby(['molecule_name', 'type'])['dist'].transform('mean')
 df[f'molecule_type_dist_mean_diff'] = df[f'molecule_type_dist_mean'] - df['dist']
return df
def dummies(df, list_cols):
for col in list_cols:
df_dummies = pd.get_dummies(df[col], drop_first=True,
prefix=(str(col)))
 df = pd.concat([df, df_dummies], axis=1)
 return df
def add_qm9_features(df):
data_qm9 = pd.read_pickle('.. /input/quantum-machine-9-qm9/data.covs.pickle')
 to_drop = ['type',
'linear',
'atom_index_0',
'atom_index_1',
'scalar_coupling_constant',
'U', 'G', 'H',
'mulliken_mean', 'r2', 'U0']
data_qm9 = data_qm9.drop(columns = to_drop, axis=1)
 data_qm9 = reduce_mem_usage(data_qm9,verbose=False)
 df = pd.merge(df, data_qm9, how='left', on=['molecule_name','id'])
 del data_qm9
df = dummies(df, ['type', 'atom_1'])
 return df
def get_features(df, struct):
for atom_idx in [0,1]:
df = map_atom_info(df, struct, atom_idx)
 df = df.rename(columns={'atom': f'atom_{atom_idx}',
'x': f'x_{atom_idx}',
'y': f'y_{atom_idx}',
'z': f'z_{atom_idx}'})
 struct['c_x'] = struct.groupby('molecule_name')['x'].transform('mean')
 struct['c_y'] = struct.groupby('molecule_name')['y'].transform('mean')
 struct['c_z'] = struct.groupby('molecule_name')['z'].transform('mean')
 df = find_dist(df)
 df = find_closest_atom(df)
 df = add_cos_features(df)
 df = add_dist_features(df)
 df = add_qm9_features(df)
 return df 
<define_variables>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
good_columns = ['type',
'dist_C_0_x',
'dist_C_1_x',
'dist_C_2_x',
'dist_C_3_x',
'dist_C_4_x',
'dist_F_0_x',
'dist_F_1_x',
'dist_F_2_x',
'dist_F_3_x',
'dist_F_4_x',
'dist_H_0_x',
'dist_H_1_x',
'dist_H_2_x',
'dist_H_3_x',
'dist_H_4_x',
'dist_N_0_x',
'dist_N_1_x',
'dist_N_2_x',
'dist_N_3_x',
'dist_N_4_x',
'dist_O_0_x',
'dist_O_1_x',
'dist_O_2_x',
'dist_O_3_x',
'dist_O_4_x',
'dist_C_0_y',
'dist_C_1_y',
'dist_C_2_y',
'dist_C_3_y',
'dist_C_4_y',
'dist_F_0_y',
'dist_F_1_y',
'dist_F_2_y',
'dist_F_3_y',
'dist_F_4_y',
'dist_H_0_y',
'dist_H_1_y',
'dist_H_2_y',
'dist_H_3_y',
'dist_H_4_y',
'dist_N_0_y',
'dist_N_1_y',
'dist_N_2_y',
'dist_N_3_y',
'dist_N_4_y',
'dist_O_0_y',
'dist_O_1_y',
'dist_O_2_y',
'dist_O_3_y',
'dist_O_4_y',
'dist_inv2',
'distance_closest_0',
'distance_closest_1',
'distance_farthest_0',
'distance_farthest_1',
'cos_c0_c1', 'cos_f0_f1','cos_c0_f0', 'cos_c1_f1',
'cos_center0_center1', 'cos_c0', 'cos_c1', 'cos_f0', 'cos_f1',
'cos_center0', 'cos_center1',
'molecule_atom_index_0_dist_mean',
'molecule_atom_index_0_dist_mean_diff',
'molecule_atom_index_0_dist_min',
'molecule_atom_index_0_dist_min_diff',
'molecule_atom_index_0_dist_std',
'molecule_atom_index_1_dist_mean',
'molecule_atom_index_1_dist_mean_diff',
'molecule_atom_index_1_dist_min',
'molecule_atom_index_1_dist_min_diff',
'molecule_atom_index_1_dist_std',
'molecule_type_dist_mean',
'molecule_type_dist_mean_diff',
'rc_A',
'rc_B',
'rc_C',
'mu',
'alpha',
'homo',
'lumo',
'gap',
'zpve',
'Cv',
'freqs_min',
'freqs_max',
'freqs_mean',
'mulliken_min',
'mulliken_max',
'mulliken_atom_0',
'mulliken_atom_1']
giba_columns = ['inv_dist0',
'inv_dist1',
'inv_distP',
'inv_dist0R',
'inv_dist1R',
'inv_distPR',
'inv_dist0E',
'inv_dist1E',
'inv_distPE',
'linkM0',
'linkM1',
'min_molecule_atom_0_dist_xyz',
'mean_molecule_atom_0_dist_xyz',
'max_molecule_atom_0_dist_xyz',
'sd_molecule_atom_0_dist_xyz',
'min_molecule_atom_1_dist_xyz',
'mean_molecule_atom_1_dist_xyz',
'max_molecule_atom_1_dist_xyz',
'sd_molecule_atom_1_dist_xyz',
'coulomb_C.x',
'coulomb_F.x',
'coulomb_H.x',
'coulomb_N.x',
'coulomb_O.x',
'yukawa_C.x',
'yukawa_F.x',
'yukawa_H.x',
'yukawa_N.x',
'yukawa_O.x',
'vander_C.x',
'vander_F.x',
'vander_H.x',
'vander_N.x',
'vander_O.x',
'coulomb_C.y',
'coulomb_F.y',
'coulomb_H.y',
'coulomb_N.y',
'coulomb_O.y',
'yukawa_C.y',
'yukawa_F.y',
'yukawa_H.y',
'yukawa_N.y',
'yukawa_O.y',
'vander_C.y',
'vander_F.y',
'vander_H.y',
'vander_N.y',
'vander_O.y',
'distC0',
'distH0',
'distN0',
'distC1',
'distH1',
'distN1',
'adH1',
'adH2',
'adH3',
'adH4',
'adC1',
'adC2',
'adC3',
'adC4',
'adN1',
'adN2',
'adN3',
'adN4',
'NC',
'NH',
'NN',
'NF',
'NO']<categorify>
%%time
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
lbl = LabelEncoder()
lbl.fit(list(train['type'].values)+ list(test['type'].values))
train['type'] = lbl.transform(list(train['type'].values))
test['type'] = lbl.transform(list(test['type'].values))<choose_model_class>
%%time
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
n_fold = 3
seed = round(time.time())
 folds = KFold(n_splits=n_fold, shuffle=True, random_state=seed)
 params = {'num_leaves': 400,
'objective': 'huber',
'max_depth': 9,
'learning_rate': 0.12,
"boosting_type": "gbdt",
"subsample_freq": 1,
"subsample": 0.8,
"metric": 'mae',
"verbosity": -1,
'lambda_l1': 0.8,
'lambda_l2': 0.2,
'feature_fraction': 0.6,
}<create_dataframe>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect()
X_short = pd.DataFrame({'ind': list(train.index),
'type': train['type'].values,
'oof': [0] * len(train),
'target': y.values})
 X_short_test = pd.DataFrame({'ind': list(test.index),
'type': test['type'].values,
'prediction': [0] * len(test)} )<prepare_x_and_y>
from sklearn.model_selection import KFold
import lightgbm as lgb
%%time
CV_score = 0
n_estimators = [5000, 2500, 3000, 2500, 2500, 3000, 2500, 2500]
for t in train['type'].unique() :
type_ = lbl.inverse_transform([t])[0]
print(f'
Training of type {t}: {type_}.')
 index_type =(train['type'] == t)
 index_type_test =(test['type'] == t)
 X_t = train.loc[index_type].copy()
X_test_t = test.loc[index_type_test].copy()
y_t = y[index_type]
print(f'Generating features...')
 start_time = time.time()
X_t = get_features(X_t, structures.copy())
 X_t = X_t[good_columns].fillna(0.0)
 X_test_t = get_features(X_test_t, structures.copy())
 X_test_t = X_test_t[good_columns].fillna(0.0)
 rows_to_exclude = np.where(index_type==False)[0]+1
rows_to_exclude_test = np.where(index_type_test==False)[0]+1
train_giba_t = pd.read_csv('.. /input/giba-molecular-features/train_giba.csv/train_giba.csv',
header=0, skiprows=rows_to_exclude, usecols=giba_columns)
 test_giba_t = pd.read_csv('.. /input/giba-molecular-features/test_giba.csv/test_giba.csv',
header=0, skiprows=rows_to_exclude_test, usecols=giba_columns)
 X_t = pd.concat(( X_t, train_giba_t), axis=1)
 X_test_t = pd.concat(( X_test_t,test_giba_t), axis=1)
 del train_giba_t, test_giba_t
gc.collect()
print(f'Done in {(time.time() - start_time):.2f} seconds for {X_t.shape[1]} features.')
 result_dict_lgb = train_model_regression(X=X_t, X_test=X_test_t,
y=y_t, params=params,
folds=folds,
model_type='lgb',
eval_metric='mae',
plot_feature_importance=False,
verbose=2000, early_stopping_rounds=200,
n_estimators=n_estimators[t])
 del X_t, X_test_t
gc.collect()
X_short.loc[X_short['type'] == t, 'oof'] = result_dict_lgb['oof']
X_short_test.loc[X_short_test['type'] == t, 'prediction'] = result_dict_lgb['prediction']
CV_score += np.log(np.array(result_dict_lgb['scores'] ).mean())/8<save_to_csv>
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
}
sub['scalar_coupling_constant'] = X_short_test['prediction']
today = str(datetime.date.today())
 sub.to_csv(f'LGB_{today}_{CV_score:.4f}.csv', index=False )<save_to_csv>
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )
sub1['scalar_coupling_constant'] = 0.30*sub1['scalar_coupling_constant'] + 0.20*sub2['scalar_coupling_constant'] + 0.30*sub3['scalar_coupling_constant'] + 0.10*sub4['scalar_coupling_constant'] + 0.10*sub5['scalar_coupling_constant']
sub1.to_csv('submission1236.csv', index=False )<set_options>
sub['isFraud'] = y_preds
sub.to_csv("submission.csv", index=False )
<define_variables><EOS> 
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;