Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<feature_engineering>
print(os.listdir(".. /input"))
os.environ['USER'] = 'root'
os.system('pip install.. /input/xlearn/xlearn/xlearn-0.40a1/')
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))<compute_test_metric>
dtypes = {
'Type' :'int32',
'Name' : 'category',
'Age' : 'float16',
'Breed1' : 'category',
'Breed2' : 'category',
'Gender' : 'category',
'Color1':'category',
'Color2' : 'category',
'Color3' : 'category',
'MaturitySize' : 'category',
'FurLength':'category',
'Vaccinated':'category',
'Dewormed':'category',
'Sterilized':'category',
'Health':'category',
'Quantity':'int16',
'Fee':'int32',
'State':'category',
'RescuerID' : 'category',
'VideoAmt':'int16',
'PhotoAmt': 'int16','Description':'object','PetID':'category','AdoptionSpeed':'int'
}
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )<merge>
from tqdm import tqdm_notebook as tqdm
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )<drop_column>
train_path = '.. /input/petfinder-adoption-prediction/train/train.csv'
dont_use = ['RescuerID', 'AdoptionSpeed', 'Description', 'PetID']
num_cols = ['Age', 'Fee', 'PhotoAmt', 'VideoAmt']
too_many_vals = [""]
categories = [k for k, v in dtypes.items() if k not in dont_use]
categories_index = dict(zip(categories, range(len(categories))))
print(categories)
 field_features = defaultdict()
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=15):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full<define_variables>
max_val = 1
with open('train.libffm', 'a')as the_file:
for t, row in tqdm(enumerate(DictReader(open(train_path)))) :
if t % 1000 == 0:
print(t, len(field_features), max_val)
 label = [row['AdoptionSpeed']]
ffeatures = []
for field in categories:
if field == 'AdoptionSpeed':
continue
feature = row[field]
if feature == '':
feature = "unk"
if field not in num_cols:
ff = field + '_____' + feature
else:
if feature == "unk" or float(feature)== -1:
ff = field + '_____' + str(0)
 else:
if field in too_many_vals:
ff = field + '_____' + str(int(round(math.log(1 + float(feature)))))
 else:
ff = field + '_____' + str(int(round(float(feature))))
if ff not in field_features:
if len(field_features)== 0:
field_features[ff] = 1
max_val += 1
else:
field_features[ff] = max_val + 1
max_val += 1
fnum = field_features[ff]
ffeatures.append('{}:{}:1'.format(categories_index[field], fnum))
line = label + ffeatures
the_file.write('{}
'.format(' '.join(line)) )
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]<choose_model_class>
test_path = '.. /input/petfinder-adoption-prediction/test/test.csv'
with open('test.libffm', 'a')as the_file:
for t, row in tqdm(enumerate(DictReader(open(test_path)))) :
if t % 1000 == 0:
print(t, len(field_features), max_val)
 label = [str(0)]
ffeatures = []
for field in categories:
if field == 'AdoptionSpeed':
continue
feature = row[field]
if feature == '':
feature = "unk"
if field not in num_cols:
ff = field + '_____' + feature
else:
if feature == "unk" or float(feature)== -1:
ff = field + '_____' + str(0)
 else:
if field in too_many_vals:
ff = field + '_____' + str(int(round(math.log(1 + float(feature)))))
 else:
ff = field + '_____' + str(int(round(float(feature))))
if ff not in field_features:
if len(field_features)== 0:
field_features[ff] = 1
max_val += 1
else:
field_features[ff] = max_val + 1
max_val += 1
fnum = field_features[ff]
ffeatures.append('{}:{}:1'.format(categories_index[field], fnum))
line = label + ffeatures
the_file.write('{}
'.format(' '.join(line)) )
def create_nn_model(input_shape):
inp = Input(shape=(input_shape,))
x = Dense(2048, activation="relu" )(inp)
 x = BatchNormalization()(x)
 x = Dense(1024, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(1024, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation="relu" )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation="relu" )(x)
 x = BatchNormalization()(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inp, outputs=[out])
 return model<set_options>
ffm_model = xl.create_ffm()
ffm_model.setTrain("train.libffm" )
config = tf.ConfigProto(device_count = {'GPU': 1 , 'CPU': 2})
 config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.6
sess = tf.Session(config=config)
 K.set_session(sess )<define_variables>
%%time
param = {'task':'reg', 'lr':0.2,
'lambda':0.002, 'metric':'rmse', 'epoch' : 50}
ffm_model.fit(param, 'xl.out' )
mol_types=train_csv["type"].unique()
cv_score=[]
cv_score_total=0
epoch_n = 1000
verbose = 1
batch_size = 2048
retrain =True
start_time=datetime.now()
test_prediction=np.zeros(len(test_csv))
input_features = ['atom_2', 'atom_3', 'atom_4', 'atom_5', 'atom_6', 'atom_7',
'atom_8','atom_9', 'atom_10','d_1_0', 'd_2_0', 'd_2_1', 'd_3_0',
'd_3_1', 'd_3_2', 'd_4_0', 'd_4_1', 'd_4_2', 'd_4_3', 'd_5_0',
'd_5_1', 'd_5_2', 'd_5_3', 'd_6_0', 'd_6_1', 'd_6_2', 'd_6_3',
'd_7_0', 'd_7_1', 'd_7_2', 'd_7_3', 'd_8_0', 'd_8_1', 'd_8_2',
'd_8_3', 'd_9_0', 'd_9_1', 'd_9_2', 'd_9_3', 'd_10_0', 'd_10_1', 'd_10_2',
'd_10_3']
for mol_type in mol_types:
model_name_wrt =('/kaggle/working/molecule_model_%s.hdf5' % mol_type)
 print('Training %s' % mol_type, 'out of', mol_types, '
')
 full = build_couple_dataframe(train_csv, structures_csv, mol_type, n_atoms=11)
 full2 = build_couple_dataframe(test_csv, structures_csv, mol_type, n_atoms=11)
 df_train_ = take_n_atoms(full, 11)
 df_test_ = take_n_atoms(full2, 11)
 df_train_ = df_train_.fillna(0)
 df_test_ = df_test_.fillna(0)
 input_data=StandardScaler().fit_transform(pd.concat([df_train_.loc[:,input_features],df_test_.loc[:,input_features]]))
target_data=df_train_.loc[:,"scalar_coupling_constant"].values
train_index, cv_index = train_test_split(np.arange(len(df_train_)) ,random_state=111, test_size=0.1)
 train_target=target_data[train_index]
cv_target=target_data[cv_index]
train_input=input_data[train_index]
cv_input=input_data[cv_index]
test_input=input_data[len(df_train_):,:]
nn_model=create_nn_model(train_input.shape[1])
 if not retrain:
nn_model = load_model(model_name_rd)
 nn_model.compile(loss='mae', optimizer=Adam())
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=40,verbose=1, mode='auto', restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.08,patience=28, min_lr=1e-6, mode='auto', verbose=1)
 sv_mod = callbacks.ModelCheckpoint(model_name_wrt, monitor='val_loss', save_best_only=True, period=1)
 history = nn_model.fit(train_input,[train_target],
validation_data=(cv_input,[cv_target]),
callbacks=[es, rlr, sv_mod], epochs=epoch_n, batch_size=batch_size, verbose=verbose)
 cv_predict=nn_model.predict(cv_input)
 plot_history(history, mol_type)
 accuracy=np.mean(np.abs(cv_target-cv_predict[:,0]))
print(np.log(accuracy))
cv_score.append(np.log(accuracy))
cv_score_total+=np.log(accuracy)
 test_predict=nn_model.predict(test_input)
 test_prediction[test_csv["type"]==mol_type]=test_predict[:,0]
K.clear_session()
cv_score_total/=len(mol_types)
<train_model>
ffm_model.setTest("test.libffm")
 ffm_model.predict("xl.out", "output_new.txt" )
print('Total training time: ', datetime.now() - start_time)
 i=0
for mol_type in mol_types:
print(mol_type,": cv score is ",cv_score[i])
 i+=1
print("total cv score is",cv_score_total )<save_to_csv>
sample = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv')
 output = pd.read_csv('output_new.txt', header=None)[0].values
plt.hist(output);
def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv("/kaggle/working/submission.csv", index=False)
 submits(test_prediction )<import_modules>
def predict(X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage<define_variables>
Coefficients = [1.645, 2.115, 2.51, 2.845]
output = predict(output, Coefficients )
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train_v2/'
TEST = '.. /input/airbus-ship-detection/test_v2/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations_v2.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-on-ship-detection-new-data/ship_detection.csv'<choose_model_class>
sample.AdoptionSpeed = output
sample['AdoptionSpeed'] = sample['AdoptionSpeed'].astype(int)
 sample.to_csv('submission.csv', index=False )
<load_from_csv><EOS> 
Counter(output )