Kaggle competition: Predicting Molecular Properties
<SOS> metric: GroupMeanLogMAE Kaggle data source: predicting-molecular-properties<load_from_csv>
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}
mo = pd.read_csv('.. /input/masseyordinals_thru_2019_day_128/MasseyOrdinals_thru_2019_day_128.csv')
 mo = mo[(mo['RankingDayNum'] == 128)&(mo['Season'] >= 2003)]
def delta_ord(row):
cond =(mo['Season'] == row['Season'])
 cond1 =(mo['TeamID'] == row['Team1'])& cond
cond2 =(mo['TeamID'] == row['Team2'])& cond
t1 = mo[cond1]['OrdinalRank'].mean()
t2 = mo[cond2]['OrdinalRank'].mean()
return t1-t2
training_set['deltaMO'] = training_set.apply(delta_ord,axis=1)
<load_from_csv>
%matplotlib inline
season_dresults = pd.read_csv('.. /input/stage2datafiles/RegularSeasonDetailedResults.csv')
 record = pd.DataFrame({'wins': season_dresults.groupby(['Season','WTeamID'] ).size() } ).reset_index() ;
losses = pd.DataFrame({'losses': season_dresults.groupby(['Season','LTeamID'] ).size() } ).reset_index() ;
record = record.merge(losses, how='outer', left_on=['Season','WTeamID'], right_on=['Season','LTeamID'])
 record = record.fillna(0)
 record['games'] = record['wins'] + record['losses']
def delta_winPct(row):
cond1 =(record['Season'] == row['Season'])&(record['WTeamID'] == row['Team1'])
 cond2 =(record['Season'] == row['Season'])&(record['WTeamID'] == row['Team2'])
 return(record[cond1]['wins']/record[cond1]['games'] ).mean() -(record[cond2]['wins']/record[cond2]['games'] ).mean()
training_set['deltaWinPct'] = training_set.apply(delta_winPct,axis=1)
<groupby>
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 120)
 pd.set_option('display.max_columns', 120 )
dfW = season_dresults.groupby(['Season','WTeamID'] ).sum().reset_index()
dfL = season_dresults.groupby(['Season','LTeamID'] ).sum().reset_index()
def get_points_for(row):
wcond =(dfW['Season'] == row['Season'])&(dfW['WTeamID'] == row['WTeamID'])
 fld1 = 'WScore'
lcond =(dfL['Season'] == row['Season'])&(dfL['LTeamID'] == row['WTeamID'])
 fld2 = 'LScore'
retVal = dfW[wcond][fld1].sum()
if len(dfL[lcond][fld2])> 0:
retVal = retVal + dfL[lcond][fld2].sum()
return retVal
def get_points_against(row):
wcond =(dfW['Season'] == row['Season'])&(dfW['WTeamID'] == row['WTeamID'])
 fld1 = 'LScore'
lcond =(dfL['Season'] == row['Season'])&(dfL['LTeamID'] == row['WTeamID'])
 fld2 = 'WScore'
retVal = dfW[wcond][fld1].sum()
if len(dfL[lcond][fld2])> 0:
retVal = retVal + dfL[lcond][fld2].sum()
return retVal
record['PointsFor'] = record.apply(get_points_for, axis=1)
 record['PointsAgainst'] = record.apply(get_points_against, axis=1 )<feature_engineering>
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )
def get_remaining_stats(row, field):
wcond =(dfW['Season'] == row['Season'])&(dfW['WTeamID'] == row['WTeamID'])
 fld1 = 'W' + field
lcond =(dfL['Season'] == row['Season'])&(dfL['LTeamID'] == row['WTeamID'])
 fld2 = 'L'+ field
retVal = dfW[wcond][fld1].sum()
if len(dfL[lcond][fld2])> 0:
retVal = retVal + dfL[lcond][fld2].sum()
return retVal
cols = ['FGM','FGA','FGM3','FGA3','FTM','FTA','OR','DR','Ast','TO','Stl','Blk','PF']
for col in cols:
print("Processing",col)
 record[col] = record.apply(get_remaining_stats, args=(col,), axis=1)
<feature_engineering>
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )
def delta_stat(row, field):
cond1 =(record['Season'] == row['Season'])&(record['WTeamID'] == row['Team1'])
 cond2 =(record['Season'] == row['Season'])&(record['WTeamID'] == row['Team2'])
 return(record[cond1][field]/record[cond1]['games'] ).mean() -(record[cond2][field]/record[cond2]['games'] ).mean()
cols = ['PointsFor','PointsAgainst','FGM','FGA','FGM3','FGA3','FTM','FTA','OR','DR','Ast','TO','Stl','Blk','PF']
for col in cols:
print("Processing",col)
 training_set['delta' + col] = training_set.apply(delta_stat,args=(col,),axis=1)
 training_set.describe() 
<import_modules>
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )
cols = ['deltaSeed','deltaMO', 'deltaWinPct','deltaPointsFor','deltaPointsAgainst','deltaFGM','deltaFGA','deltaFGM3','deltaFGA3','deltaFTM',
'deltaFTA','deltaOR','deltaDR','deltaAst','deltaTO','deltaStl','deltaBlk','deltaPF']
X = training_set[cols]
y = training_set['Result']
logit_model=sm.Logit(y,X)
 result=logit_model.fit()
print(result.summary2())
<define_search_space>
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )
winner=[0.52951,0.438933,0.481309,0.438575,0.531942]
top5=[0.54772,0.453872,0.519706,0.461073,0.549876]
topprctvalues=[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100]
topprctscores=[[0.55620,0.465207,0.536976,0.472893,0.576667,],
[0.56035,0.469617,0.543597,0.479488,0.581399,],
[0.56411,0.474137,0.550067,0.486432,0.586220,],
[0.56958,0.478107,0.556954,0.491655,0.590554,],
[0.57177,0.481519,0.563566,0.495595,0.594101,],
[0.57495,0.486253,0.569619,0.500194,0.597493,],
[0.57946,0.490536,0.577442,0.505821,0.601023,],
[0.58676,0.496779,0.582541,0.511053,0.603920,],
[0.59418,0.500862,0.585897,0.515844,0.608200,],
[0.59921,0.508830,0.589459,0.522958,0.613297,],
[0.60593,0.516735,0.594342,0.528533,0.621995,],
[0.61288,0.523082,0.607036,0.534732,0.631227,],
[0.61915,0.528358,0.627708,0.541466,0.645569,],
[0.62780,0.539478,0.646980,0.556034,0.664821,],
[0.65659,0.552809,0.682279,0.576019,0.688618,],
[0.67932,0.574223,0.764058,0.605337,0.705595,],
[0.73514,0.625012,0.865465,0.643514,0.776256,],
[0.98764,0.692878,0.865465,0.693147,1.071796],
[1.59656,0.882637,1.045207,0.897480,1.204277,],
[19.18820,11.513103,9.092529,12.609623,18.092158,],] 
<compute_train_metric>
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures
cols = ['deltaMO', 'deltaWinPct', 'deltaPointsAgainst']
biases = {
'1163':1.2,
'1181':1.2,
'1437':1.2,
'1314':1.2,
}
errs = []
for year in range(2003,2019):
print("Evaluation on tournament year",year)
 Xtrain = training_set[training_set['Season'] != year][cols]
ytrain = training_set[training_set['Season'] != year]['Result']
logit_model=sm.Logit(ytrain,Xtrain)
 result=logit_model.fit()
Xtest = training_set[training_set['Season'] == year][cols]
ytest = training_set[training_set['Season'] == year]['Result']
pred = result.predict(Xtest)
 for bias in biases:
pred.loc[(training_set['Season'] == year)&(training_set['Team1'] == int(bias)) ] = pred.loc[(training_set['Season'] == year)&(training_set['Team1'] == int(bias)) ] * biases[bias]
pred.loc[(training_set['Season'] == year)&(training_set['Team2'] == int(bias)) ] = pred.loc[(training_set['Season'] == year)&(training_set['Team2'] == int(bias)) ] / biases[bias]
pred.loc[(pred >= 0.9999)] = 0.9999
pred.loc[(pred <= 0.0001)] = 0.0001
pred.loc[training_set[training_set['Season'] == year]['Result'] == 0] = 1 - pred.loc[training_set[training_set['Season'] == year]['Result'] == 0]
err = -np.log(pred ).mean()
errs.append(err)
 print("Mean log loss: ",np.mean(errs))
print("Log losses by season")
 print("--------------------")
 print("year","your score",sep="\t")
 years1 = range(2003,2014)
 for i in range(len(years1)) :
print(years1[i], "{0:.6f}".format(errs[i]),sep="\t")
 print("Log losses by season")
 print("--------------------")
 print("year","your score","your result","winning score",sep="\t")
 years2 = range(2014,2019)
 for i in range(len(years2)) :
result = None
if errs[i+len(years1)] < winner[i]:
result="Win competition"
elif errs[i+len(years1)] < top5[i]:
result="Top 5 score"
else:
for j in range(len(topprctvalues)) :
if errs[i+len(years1)] < topprctscores[j][i]:
result = "Top " + str(topprctvalues[j])+ "%"
break
if result is None:
result = "Worst score on Kaggle"
print(years2[i], "{0:.6f}".format(errs[i+len(years1)]),result+"\t",winner[i],sep="\t")
<feature_engineering>
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df
sub = pd.read_csv('.. /input/SampleSubmissionStage2.csv')
 sub['Season'], sub['Team1'], sub['Team2'] = sub['ID'].str.split('_' ).str
sub[['Season', 'Team1', 'Team2']] = sub[['Season', 'Team1', 'Team2']].apply(pd.to_numeric)
 sub['deltaSeed'] = sub.apply(delta_seed,axis=1)
 sub['deltaMO'] = sub.apply(delta_ord,axis=1)
 sub['deltaWinPct'] = sub.apply(delta_winPct,axis=1)
 cols = ['PointsFor','PointsAgainst','FGM','FGA','FGM3','FGA3','FTM','FTA','OR','DR','Ast','TO','Stl','Blk','PF']
for col in cols:
print("Processing",col)
 sub['delta' + col] = sub.apply(delta_stat,args=(col,),axis=1)
 cols = ['deltaMO', 'deltaWinPct', 'deltaPointsAgainst']
biases = {
'1163':1.2,
'1181':1.2,
'1437':1.2,
'1314':1.2,
}
Xtrain = training_set[cols]
ytrain = training_set['Result']
logit_model=sm.Logit(ytrain,Xtrain)
 result=logit_model.fit()
Xtest = sub[cols]
pred = result.predict(Xtest)
 for bias in biases:
pred.loc[sub['Team1'] == int(bias)] = pred.loc[sub['Team1'] == int(bias)] * biases[bias]
pred.loc[sub['Team2'] == int(bias)] = pred.loc[sub['Team2'] == int(bias)] / biases[bias]
pred.loc[(pred >= 0.9999)] = 0.9999
pred.loc[(pred <= 0.0001)] = 0.0001
pred.loc[(sub['Team1'] == 1163)&(sub['Team2'] == 1181)] = 0.8
sub['Pred'] = pred
sub[['ID', 'Pred']].to_csv('submission_lr.csv', index=False)
<train_model>
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df
model = en.ExtraTreesClassifier(n_estimators=200)
 model.fit(Xtrain.fillna(-1), ytrain)
 predictions = model.predict(Xtrain.fillna(-1)).clip(0,1)
 print('Log Loss:', log_loss(ytrain, predictions))<save_to_csv>
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df
predictions = model.predict(Xtest.fillna(-1)).clip(0,1)
 sub['Pred'] = predictions
sub[['ID', 'Pred']].to_csv('submission_et.csv', index=False )<load_pretrained>
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))
FileLink('./submission_et.csv' )<load_from_csv>
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )
%matplotlib inline
plt.rcParams['figure.figsize'] =(15, 9)
 season_df = pd.read_csv('.. /input/datafiles/RegularSeasonCompactResults.csv')
 tourney_df = pd.read_csv('.. /input/datafiles/NCAATourneyCompactResults.csv')
 ordinals_df = pd.read_csv('.. /input/masseyordinals/MasseyOrdinals.csv')\ 
.rename(columns={'RankingDayNum':'DayNum'})
 ordinals_df = ordinals_df.groupby(['SystemName','Season','TeamID'] ).last().reset_index()
ordinals_df.head()<merge>
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]
games_df = tourney_df.merge(ordinals_df,left_on=['Season','WTeamID'],
right_on=['Season','TeamID'])
 games_df.head()
games_df = games_df.merge(ordinals_df,left_on=['Season','LTeamID','SystemName'],
right_on=['Season','TeamID','SystemName'],
suffixes = ['W','L'])
 games_df.head() 
<sort_values>
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full
results_by_system['prediction'].sort_values('mean',ascending=False)[:50]<feature_engineering>
%%time
def type_select(types = '1JHN', n_atoms=10):
full = build_couple_dataframe(train_csv, structures_csv, types, n_atoms=n_atoms)
 print(full.shape)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 X_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.1, random_state=128)
 print(X_train.shape, X_val.shape, y_train.shape, y_val.shape)
 return X_train, X_val, y_train, y_val
games_df['Wrating'] = 100-4*np.log(games_df['OrdinalRankW']+1)-games_df['OrdinalRankW']/22
games_df['Lrating'] = 100-4*np.log(games_df['OrdinalRankL']+1)-games_df['OrdinalRankL']/22
games_df['prob'] = 1/(1+10**(( games_df['Lrating']-games_df['Wrating'])/15))
loss_results = games_df[games_df.Season>=2015].groupby('SystemName')['prob'].agg([('loss',lambda p: -np.mean(np.log(p))),('count','count')])
 loss_results['loss'].sort_values() [:50].plot.bar(ylim=[.4,.6] )<feature_engineering>
X_train, X_val, y_train, y_val = type_select(types = '3JHN')
ref_system = 'POM'
ordinals_df['Rating']= 100-4*np.log(ordinals_df['OrdinalRank']+1)-ordinals_df['OrdinalRank']/22
ordinals_df = ordinals_df[ordinals_df.SystemName==ref_system]
submission_df = pd.read_csv('.. /input/SampleSubmissionStage1.csv')
 submission_df['Season'] = submission_df['ID'].map(lambda x: int(x.split('_')[0]))
submission_df['Team1'] = submission_df['ID'].map(lambda x: int(x.split('_')[1]))
submission_df['Team2'] = submission_df['ID'].map(lambda x: int(x.split('_')[2]))
submission_df = submission_df.merge(ordinals_df[['Season','TeamID','Rating']],how='left',
left_on = ['Season','Team1'], right_on = ['Season','TeamID'])
 submission_df = submission_df.merge(ordinals_df[['Season','TeamID','Rating']],how='left',
left_on = ['Season','Team2'], right_on = ['Season','TeamID'],
suffixes=['W','L'])
 submission_df['Pred'] = 1/(1+10**(( submission_df['RatingL']-submission_df['RatingW'])/15))
submission_df[['ID', 'Pred']].to_csv('submission.csv', index=False)
 submission_df[['ID', 'Pred']].head()<install_modules>
model_parameters = {'n_estimators': [50, 100, 150, 200, 250, 300],
'max_depth':[5,7,9,11], 'learning_rate': [0.07, 0.1, 0.15],
'gamma': [0, 0.0001, 0.001, 0.01],
'subsample': [0.5, 0.8, 1], 'colsample_bytree': [0.5, 0.66, 1]
}
fit_params = {'eval_metric': 'mae',
'early_stopping_rounds': 5,
'eval_set': [(X_val, y_val)]}
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
!pip install imgaug<import_modules>
best_parameters = {'base_score': 0.5,
'booster': 'gbtree',
'colsample_bylevel': 1,
'colsample_bynode': 1,
'colsample_bytree': 1,
'gamma': 0,
'importance_type': 'gain',
'learning_rate': 0.15,
'max_delta_step': 0,
'max_depth': 11,
'min_child_weight': 1,
'missing': None,
'n_estimators': 300,
'n_jobs': 4,
'nthread': None,
'objective': 'reg:squarederror',
'random_state': 0,
'reg_alpha': 0,
'reg_lambda': 1,
'scale_pos_weight': 1,
'seed': None,
'silent': None,
'subsample': 0.8,
'verbosity': 1
}
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from tqdm import tqdm_notebook
import matplotlib.pyplot as plt
import random
import math
import imgaug as ia
from imgaug import augmenters as iaa<define_variables>
PATH = './'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
LABELS = '.. /input/humpback-whale-identification/train.csv'
BOXES = '.. /input/generating-whale-bounding-boxes/bounding_boxes.csv'
MODLE_INIT = '.. /input/pytorch-pretrained-models/'
n_embedding = 512
bs = 32
sz = 224
nw = 2<train_on_grid>
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data
def open_image(fn):
flags = cv2.IMREAD_UNCHANGED+cv2.IMREAD_ANYDEPTH+cv2.IMREAD_ANYCOLOR
if not os.path.exists(fn):
raise OSError('No such file or directory: {}'.format(fn))
elif os.path.isdir(fn):
raise OSError('Is a directory: {}'.format(fn))
else:
try:
im = cv2.imread(str(fn), flags)
 if im is None: raise OSError(f'File not recognized by opencv: {fn}')
 return cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
 except Exception as e:
raise OSError('Error handling image at: {}'.format(fn)) from e
class Loader() :
def __init__(self,path,tfms_g=None, tfms_px=None):
self.boxes = pd.read_csv(BOXES ).set_index('Image')
 self.path = path
self.tfms_g = iaa.Sequential(tfms_g,random_order=False)\
if tfms_g is not None else None
self.tfms_px = iaa.Sequential(tfms_px,random_order=False)\
if tfms_px is not None else None
def __call__(self, fname):
fname = os.path.basename(fname)
 x0,y0,x1,y1 = tuple(self.boxes.loc[fname,['x0','y0','x1','y1']].tolist())
 img = open_image(os.path.join(self.path,fname))
l1,l0,_ = img.shape
b0,b1 = x1-x0, y1-y0
x0n,x1n = max(int(x0 - b0*0.05),0), min(int(x1 + b0*0.05),l0-1)
 y0n,y1n = max(int(y0 - b1*0.05),0), min(int(y1 + b1*0.05),l1-1)
 if self.tfms_g != None: img = self.tfms_g.augment_image(img)
 img = cv2.resize(img[y0n:y1n,x0n:x1n,:],(sz,sz))
if self.tfms_px != None: img = self.tfms_px.augment_image(img)
 return img.astype(np.float)/255<feature_engineering>
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 model_ = XGBRegressor(**best_parameters)
 model_.fit(X_data, y_data)
 y_pred += model_.predict(X_test)
 submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
def get_idxs0(names, df, n=64):
idxs = []
for name in names:
label = df[df.Image == name].Id.values[0]
idxs.append(df[df.Id != label].Image.sample(n ).values)
 return idxs
class Image_selection:
def __init__(self,fnames,data,emb_df=None,model=None):
if emb_df is None or model is None:
df = data.copy()
counts = Counter(df.Id.values)
 df['c'] = df['Id'].apply(lambda x: counts[x])
 df['label'] = df.Id
df.loc[df.c == 1,'label'] = 'new_whale'
df = df.sort_values(by=['c'])
 df.label = pd.factorize(df.label)[0]
l1 = 1 + df.label.max()
l2 = len(df[df.label==0])
 df.loc[df.label==0,'label'] = range(l1, l1+l2)
 df = df.set_index('label')
 l = len(fnames)
 idxs = Parallel(n_jobs=nw )(delayed(get_idxs0)\ 
(fnames[int(i*l/nw):int(( i+1)*l/nw)], df)for i in range(nw))
idxs = [y for x in idxs for y in x]
else:
data = data.copy().set_index('Image')
 trn_emb = emb_df.copy()
trn_emb.set_index('files',inplace=True)
 trn_emb['emb'] = [[float(i)for i in s.split() ] for s in trn_emb['emb']]
trn_emb = data.join(trn_emb)
 trn_emb = trn_emb.reset_index()
trn_emb['idx'] = np.arange(len(trn_emb))
trn_emb = trn_emb.set_index('Id')
 emb = np.array(trn_emb.emb.tolist())
 l = len(fnames)
 idxs = []
model.eval()
with torch.no_grad() :
m = model.module if isinstance(model,FP16)else model
emb = torch.from_numpy(emb ).half().cuda()
for name in tqdm_notebook(fnames):
label = trn_emb.loc[trn_emb.Image == name].index[0]
v0 = np.array(trn_emb.loc[trn_emb.Image == name,'emb'].tolist() [0])
 v0 = torch.from_numpy(v0 ).half().cuda()
d = m.get_d(v0,emb)
 ids = trn_emb.loc[trn_emb.index!=label].idx.tolist()
sorted, indices = torch.sort(d[ids])
 idxs.append([ids[i] for i in indices[:64]])
 trn_emb = trn_emb.set_index('idx')
 idxs = [trn_emb.loc[idx,'Image'] for idx in idxs]
self.df = pd.DataFrame({'Image':fnames,'idxs':idxs} ).set_index('Image')
 def get(self,name):
return np.random.choice(self.df.loc[name].values[0],1)[0]<feature_engineering>
submission = submission_csv.copy()
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type] )
<split><EOS> 
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )