Kaggle competition: Northeastern SMILE Lab - Recognizing Faces in the Wild
<SOS> metric: AUC Kaggle data source: northeastern-smile-lab-recognizing-faces-in-the-wild<groupby>
from collections import defaultdict
from glob import glob
from random import choice, sample
import cv2
import numpy as np
import pandas as pd
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau,EarlyStopping
from keras.layers import Input, Dense, Flatten, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract, Add, Conv2D, Flatten
from keras.models import Model
from keras.layers import BatchNormalization
from keras.preprocessing import image
from keras.optimizers import Adam
import h5py
from keras.layers import LeakyReLU
from keras import regularizers
import gc
import psutil
top_mean = 0
s = 0
for n in [0,1,3,7,26]:
top_mean += concat_sub.iloc[:,n]*scores[top[n]]
s += scores[top[n]]
top_mean /= s<save_to_csv>
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"
val_famillies = "F09"
concat_sub['isFraud'] = m_gmean
concat_sub[['isFraud']].to_csv('stack_gmean.csv' )<set_options>
all_images = glob(train_folders_path + "*/*/*.jpg")
 train_images = [x for x in all_images if val_famillies not in x]
val_images = [x for x in all_images if val_famillies in x]
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
train_person_to_images_map = defaultdict(list)
 ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]
for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x )
folder_path = '.. /input/ieee-fraud-detection/'
train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv' )<merge>
relationships = pd.read_csv(train_file_path)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl]
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 del train_identity, train_transaction, test_identity, test_transaction
gc.collect()<feature_engineering>
train = [x for x in relationships if val_famillies not in x[0]]
val = [x for x in relationships if val_famillies in x[0]]
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe<define_variables>
def read_img(path):
img = image.load_img(path, target_size=(197, 197))
img = np.array(img ).astype(np.float)
 return preprocess_input(img, version=2 )
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']<split>
def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1 = np.array([read_img(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2 = np.array([read_img(x)for x in X2])
 yield [X1, X2], labels
train = id_split(train)
 test = id_split(test )<drop_column>
def baseline_model() :
input_1 = Input(shape=(197, 197, 3))
input_2 = Input(shape=(197, 197, 3))
base_model = VGGFace(model='resnet50', include_top=False)
 for x in base_model.layers[:-3]:
x.trainable = True
x1 = base_model(input_1)
 x2 = base_model(input_2)
 x1 = Concatenate(axis=-1 )([GlobalAvgPool2D()(x1), GlobalAvgPool2D()(x1)])
 x2 = Concatenate(axis=-1 )([GlobalAvgPool2D()(x2), GlobalAvgPool2D()(x2)])
 x3 = Subtract()([x1, x2])
 x3 = Multiply()([x3, x3])
 x1_ = Multiply()([x1, x1])
 x2_ = Multiply()([x2, x2])
 x4 = Subtract()([x1_, x2_])
 x = Concatenate(axis=-1 )([x4, x3])
 x = Dense(256, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = Dropout(0.01 )(x)
 x = Dense(128, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = BatchNormalization()(x)
 x = Dense(128, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = Dropout(0.01 )(x)
 x = Dense(100, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model([input_1, input_2], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00001))
model.summary()
return model
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT')
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<categorify>
!pip install git+https://github.com/rcmalli/keras-vggface.git
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )<feature_engineering>
print("available RAM:", psutil.virtual_memory())
 gc.collect()
print("available RAM:", psutil.virtual_memory() )
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24 
<data_type_conversions>
test_path = ".. /input/test/"
<categorify><EOS> 
def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))
submission = pd.read_csv('.. /input/sample_submission.csv')
 predictions = []
for batch in tqdm(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1 = np.array([read_img(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2 = np.array([read_img(test_path + x)for x in X2])
 pred = curr_model.predict([X1, X2] ).ravel().tolist()
predictions += pred
submission['is_related'] = predictions
submission.to_csv("vgg_face.csv", index=False )