Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<define_variables>
%matplotlib inline
sub_files = ['.. /input/ensembling-m13-0-736/sub_ens.csv',
'.. /input/rxt50-448-rxt50-384-c-bb-lb-0-718/submission.csv',
'.. /input/ensembling-with-averaged-probabilties-0-701-lb/submission.csv']
sub_weight = [0.736**2,
0.718**2,
0.701**2]<save_to_csv>
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
Hlabel = 'Image'
Htarget = 'Id'
npt = 5
place_weights = {}
for i in range(npt):
place_weights[i] =(1 /(i + 1))
print(place_weights)
 lg = len(sub_files)
 sub = [None]*lg
for i, file in enumerate(sub_files):
print("Reading {}: w={} - {}".format(i, sub_weight[i], file))
reader = csv.DictReader(open(file,"r"))
sub[i] = sorted(reader, key=lambda d: str(d[Hlabel]))
out = open("sub_ens.csv", "w", newline='')
 writer = csv.writer(out)
 writer.writerow([Hlabel,Htarget])
 for p, row in enumerate(sub[0]):
target_weight = {}
for s in range(lg):
row1 = sub[s][p]
for ind, trgt in enumerate(row1[Htarget].split(' ')) :
target_weight[trgt] = target_weight.get(trgt,0)+(place_weights[ind]*sub_weight[s])
 tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:npt]
writer.writerow([row1[Hlabel], " ".join(tops_trgt)])
 out.close()<define_variables>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )
MODEL_NAME = 'Resnet50'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = resnet50
num_workers = 4<count_unique_values>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )
train_df = pd.read_csv(LABELS ).set_index('Image')
 unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]<feature_engineering>
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<prepare_x_and_y>
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels)
<categorify>
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=25, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md 
<load_pretrained>
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df
image_size = 224
batch_size = 96
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam<init_hyperparams>
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]<train_model>
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))
base_lr = 1e-4
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn.fit(lrs, n_cycle=38, cycle_len=None )<prepare_output>
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1 )<save_to_csv>
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('submission_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()<set_options>
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full
pd.options.display.max_rows = 500
pd.options.display.max_columns = 100<load_from_csv>
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]
train = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv')
 train_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 test = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')
 test_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv' )<load_from_csv>
%%time
full = build_couple_dataframe(train_csv, structures_csv, '1JHN', n_atoms=10)
 print(full.shape )
train_len = len(train)
<merge>
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 X_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.3, random_state=128)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape
train = train.merge(train_ind, how = 'left', on ='TransactionID')
 test = test.merge(test_ind, how = 'left', on ='TransactionID')
 del train_ind,test_ind<concatenate>
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data
all_data = pd.concat([train, test])
 del train,test
gc.collect()<feature_engineering>
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, n_folds=3, n_splits=5, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 cv_score = 0
if n_folds > n_splits:
n_splits = n_folds
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
if fold >= n_folds:
break
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
iterations=3000
if coupling_type in ['1JHN','1JHC']:
iterations=12000
model = CatBoostRegressor(n_estimators=iterations,
verbose=-1,
objective="MAE",
loss_function="MAE",
boosting_type="Ordered",
reg_lambda=0,
learning_rate=0.2,
task_type='GPU',
subsample=0.9,
max_depth=10,
bootstrap_type='Bernoulli',
colsample_bylevel=1 
)
 model.fit(X_train, y_train,
eval_set=[(X_val, y_val)],
verbose=100,
early_stopping_rounds=300)
 y_val_pred = model.predict(X_val)
 val_score = np.log(mean_absolute_error(y_val, y_val_pred))
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(X_test)/ n_folds
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
return cv_score
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 all_data['DT_time'] = all_data['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 all_data['count'] = 1
all_data['diff_days_from_first_transaction'] = all_data['D1'].fillna(0 ).apply(lambda x:(datetime.timedelta(days = x)))
 all_data['client_firstdate'] = all_data['DT_time']- all_data['diff_days_from_first_transaction']
all_data['client_firstdate_days'] =(all_data['DT_time']- all_data['diff_days_from_first_transaction'] ).apply(lambda x:str(x.date()))<define_variables>
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )
anaysis_fea = [ 'TransactionID',
'isFraud',
'TransactionDT',
'TransactionAmt',
'ProductCD',
'device_hash','card_hash', 'V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo',
'card1','card2','card3','card4','card5','card6','client_firstdate_days','dist1','dist2','P_emaildomain','addr1','addr2','train_or_test','count','DT_time','diff_days_from_first_transaction','client_firstdate']
drop_fea = [col for col in all_data.columns if col not in anaysis_fea]<drop_column>
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
N_FOLDS = 5
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score
<set_options><EOS> 
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )