Kaggle competition: Facial Keypoints Detection
train_data[train_data['Transmission']=='Manual']<rename_columns>
FTRAIN = '.. /input/training/training.csv'
FTEST = '.. /input/test/test.csv'
def load(test=False, cols=None):

 fname = FTEST if test else FTRAIN
df = read_csv(os.path.expanduser(fname))
df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep=' '))
if cols:
df = df[list(cols)+ ['Image']]
print(df.count())
 df = df.dropna()
X = np.vstack(df['Image'].values)/ 255.
 X = X.astype(np.float32)
 if not test:
y = df[df.columns[:-1]].values
y =(y - 48)/ 48
X, y = shuffle(X, y, random_state=42)
 y = y.astype(np.float32)
 else:
y = None
return X, y
X, y = load()
print("X.shape == {}; X.min == {:.3f}; X.max == {:.3f}".format(
 X.shape, X.min() , X.max()))
print("y.shape == {}; y.min == {:.3f}; y.max == {:.3f}".format(
 y.shape, y.min() , y.max()))
train_data['Transmission'].replace("Manual" ,"MANUAL" , inplace=True )<count_values>
def load2d(test=False,cols=None):
re = load(test, cols)
 X = re[0].reshape(-1,96,96,1)
 y = re[1]
return X, y
train_data['Transmission'].value_counts()<data_type_conversions>
%%time
model = Sequential()
model.add(Dense(128,input_dim=X.shape[1]))
model.add(Activation('relu'))
model.add(Dropout(0.1))
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(30))
model.summary()
train_data['Transmission'].fillna(value='Trans_unknown' , inplace =True)
 test_data['Transmission'].fillna(value='Trans_unknown' , inplace =True )<count_values>
sgd = SGD(lr=0.01, momentum=0.9, nesterov=True)
 model.compile(loss='mean_squared_error', optimizer=sgd)
 hist = model.fit(X, y, nb_epoch=50,batch_size=128, validation_split=0.2,verbose=False )
train_data['WheelTypeID'].value_counts()<drop_column>
X_test, _ = load(test=True)
 y_test = model.predict(X_test )
train_data.drop('WheelTypeID' , axis=1 , inplace=True)
 test_data.drop('WheelTypeID' , axis=1 , inplace=True )<count_values>
model.save('my_model.h5')
 model = load_model('my_model.h5' )
train_data['WheelType'].value_counts()<data_type_conversions>
train_data['WheelType'].fillna(value='WT_unknown' , inplace =True)
 test_data['WheelType'].fillna(value='WT_unknown' , inplace =True )<count_values>
X,y = load2d()
train_data['Nationality'].value_counts()<data_type_conversions>
from keras.layers import MaxPooling2D, Conv2D , Flatten, Dropout
from keras.layers.normalization import BatchNormalization
train_data['Nationality'].fillna(value='Nationality_unknown' , inplace =True)
 test_data['Nationality'].fillna(value='Nationality_unknown' , inplace =True )<count_values>
def CNN() :
model2 = Sequential()
model2.add(Conv2D(filters=16,kernel_size=2,padding="same",activation="relu",input_shape=(96,96,1)))
 model2.add(Dropout(0.1))
model2.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode="valid"))
model2.add(BatchNormalization())
 model2.add(Conv2D(32, 5, 5,activation="relu"))
model2.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode="valid"))
model2.add(Dropout(0.2))
model2.add(BatchNormalization())
 model2.add(Conv2D(64, 5, 5,activation="relu"))
model2.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode="valid"))
model2.add(BatchNormalization())
 model2.add(Conv2D(128, 3, 3,activation="relu"))
model2.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode="valid"))
model2.add(Dropout(0.4))
model2.add(BatchNormalization())
 model2.add(Flatten())
 model2.add(Dense(500, activation="relu"))
model2.add(Dropout(0.1))
model2.add(Dense(128, activation="relu"))
model2.add(Dropout(0.1))
model2.add(Dense(30))
model2.summary()
model2.compile(optimizer='adam',
loss='mse',
metrics=['mae','accuracy'])
 return(model2 )
train_data['Size'].value_counts()<data_type_conversions>
model2 = CNN()
hist2 = model2.fit(X, y, nb_epoch=500,batch_size=128, validation_split=0.2,verbose=False )
train_data['Size'].fillna(value='size_unknown' , inplace =True)
 test_data['Size'].fillna(value='size_unknown' , inplace =True )<count_values>
sample1,_ = load(test=True)
 sample2,_ = load2d(test=True)
 y_pred1 = model.predict(sample1)
 y_pred2 = model2.predict(sample2 )
train_data['TopThreeAmericanName'].value_counts()<data_type_conversions>
model2.save('my_model2.h5')
 model2 = load_model('my_model2.h5' )
train_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True)
 test_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True )<count_values>
class FlippedImageDataGenerator(ImageDataGenerator):
flip_indices = [ 
(0, 2),(1, 3), 
(4, 8),(5, 9),(6, 10),(7, 11), 
(12, 16),(13, 17),(14, 18),(15, 19), 
(22, 24),(23, 25),
]
def next(self):
X_batch, y_batch = super(FlippedImageDataGenerator, self ).next()
batch_size = X_batch.shape[0]
indices = np.random.choice(batch_size, batch_size/2, replace=False)
 X_batch[indices] = X_batch[indices, :, :, ::-1]
if y_batch is not None:
y_batch[indices, ::2] = y_batch[indices, ::2] * -1
for a, b in self.flip_indices:
y_batch[indices, a], y_batch[indices, b] =(
 y_batch[indices, b], y_batch[indices, a] 
)
 return X_batch, y_batch
train_data['PRIMEUNIT'].value_counts()<categorify>
X, y = load2d()
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42 )
train_data['PRIMEUNIT'].fillna(value='unk',inplace=True)
 test_data['PRIMEUNIT'].fillna(value='unk',inplace=True )<count_values>
model3 = CNN()
flipgen = FlippedImageDataGenerator()
hist3 = model3.fit_generator(flipgen.flow(X_train, y_train),
samples_per_epoch=X_train.shape[0],
nb_epoch=300,
validation_data=(X_val, y_val))
train_data['AUCGUART'].value_counts()<data_type_conversions>
model3.save('my_model3.h5')
 model3 = load_model('my_model3.h5' )
train_data['AUCGUART'].fillna(value='AUC_unk',inplace=True)
 test_data['AUCGUART'].fillna(value='AUC_unk',inplace=True )<drop_column>
SPECIALIST_SETTINGS = [
dict(
 columns=(
 'left_eye_center_x', 'left_eye_center_y',
'right_eye_center_x', 'right_eye_center_y', 
),
flip_indices=(( 0, 2),(1, 3)) , 
),
dict(
 columns=(
 'nose_tip_x', 'nose_tip_y', 
),
flip_indices=() , 
),
dict(
 columns=(
 'mouth_left_corner_x', 'mouth_left_corner_y',
'mouth_right_corner_x', 'mouth_right_corner_y',
'mouth_center_top_lip_x', 'mouth_center_top_lip_y', 
),
flip_indices=(( 0, 2),(1, 3)) , 
),
dict(
 columns=(
 'mouth_center_bottom_lip_x',
'mouth_center_bottom_lip_y', 
),
flip_indices=() , 
),
dict(
 columns=(
 'left_eye_inner_corner_x', 'left_eye_inner_corner_y',
'right_eye_inner_corner_x', 'right_eye_inner_corner_y',
'left_eye_outer_corner_x', 'left_eye_outer_corner_y',
'right_eye_outer_corner_x', 'right_eye_outer_corner_y', 
),
flip_indices=(( 0, 2),(1, 3),(4, 6),(5, 7)) , 
),
dict(
 columns=(
 'left_eyebrow_inner_end_x', 'left_eyebrow_inner_end_y',
'right_eyebrow_inner_end_x', 'right_eyebrow_inner_end_y',
'left_eyebrow_outer_end_x', 'left_eyebrow_outer_end_y',
'right_eyebrow_outer_end_x', 'right_eyebrow_outer_end_y', 
),
flip_indices=(( 0, 2),(1, 3),(4, 6),(5, 7)) , 
),
]
train_data.drop([ 'MMRAcquisitionAuctionAveragePrice', 'MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice', 'MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice', 'MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice', 'MMRCurrentRetailCleanPrice',] ,inplace=True , axis=1)
 test_data.drop([ 'MMRAcquisitionAuctionAveragePrice', 'MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice', 'MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice', 'MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice', 'MMRCurrentRetailCleanPrice',] ,inplace=True , axis=1 )<drop_column>
def fit_specialists(freeze=True,
print_every=10,
epochs=100,
prop=0.1,
name_transfer_model="my_model3.h5"):
specialists = OrderedDict()
for setting in SPECIALIST_SETTINGS:
cols = setting['columns']
flip_indices = setting['flip_indices']
X, y = load2d(cols=cols)
 X_train, X_val, y_train, y_val = train_test_split(X, y,
test_size=0.2,
random_state=42)
 model4 = load_model(name_transfer_model)
 if freeze:
for layer in model.layers:
layer.trainable = False
model4.layers.pop()
model4.outputs = [model4.layers[-1].output]
model4.layers[-1].outbound_nodes = []
model4.add(Dense(len(cols)))
 model4.compile(loss='mean_squared_error', optimizer="adam")
 flipgen = FlippedImageDataGenerator()
flipgen.flip_indices = setting['flip_indices']
print(X_train.shape)
 print(y_train.shape)
 print(X_val.shape)
 print(y_val.shape)
 hist_final = model4.fit_generator(flipgen.flow(X_train, y_train),
samples_per_epoch=X_train.shape[0],
nb_epoch=epochs,
validation_data=(X_val, y_val))
specialists[cols] = model4
return(specialists)
train_data.drop('PurchDate' ,inplace=True , axis=1)
 test_data.drop('PurchDate' ,inplace=True , axis=1 )<drop_column>
%%time
specialists1 = fit_specialists(freeze=True,
print_every=10,
epochs=100,
name_transfer_model="my_model3.h5" )
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!=object<drop_column>
X_test,_ = load2d(test=True)
 y_pred3 = model3.predict(X_test)
 landmark_nm = read_csv(os.path.expanduser(FTRAIN)).columns[:-1].values
df_y_pred3 = DataFrame(y_pred3,columns = landmark_nm)
 def predict_specialist(specialists1,X_test):
y_pred_s = []
for columns, value in specialists1.items() :
smodel = value
y_pred = smodel.predict(X_test)
 y_pred = DataFrame(y_pred,columns=columns)
 y_pred_s.append(y_pred)
 df_y_pred_s = concat(y_pred_s,axis=1)
 return(df_y_pred_s)
 df_y_pred_s = predict_specialist(specialists1,X_test)
 y_pred_s = df_y_pred_s.values
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']<feature_engineering>
FIdLookup = '.. /input/IdLookupTable.csv'
IdLookup = read_csv(os.path.expanduser(FIdLookup))
def prepare_submission(y_pred4,filename):

 ImageId = IdLookup["ImageId"]
FeatureName = IdLookup["FeatureName"]
RowId = IdLookup["RowId"]
submit = []
for rowId,irow,landmark in zip(RowId,ImageId,FeatureName):
submit.append([rowId,y_pred4[landmark].iloc[irow-1]])
 submit = DataFrame(submit,columns=["RowId","Location"])
 submit["Location"] = submit["Location"]*48 + 48
print(submit.shape)
 if filename == "model3":
submit.to_csv("model3.csv",index=False)
 else:
submit.to_csv("special.csv",index=False)
 prepare_submission(df_y_pred_s,"special")
 prepare_submission(df_y_pred3,"model3")
for i in not_categorical :
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
maximum_test=np.max(test_data[i])
 test_data[i]=test_data[i]/maximum_test 
<drop_column>
categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']<filter>
df_y_pred_s = df_y_pred_s[df_y_pred3.columns]
df_compare = {}
df_compare["difference"] =(( df_y_pred_s - df_y_pred3)**2 ).mean(axis=1)
 df_compare["RowId"] = range(df_y_pred_s.shape[0])
 df_compare = DataFrame(df_compare)
 df_compare = df_compare.sort_values("difference",ascending=False )
train_data[categorical[0]]<categorify>
def fit_specialists(freeze=True,
print_every=10,
epochs=100,
prop=0.1,
name_transfer_model="my_model2.h5"):
specialists = OrderedDict()
for setting in SPECIALIST_SETTINGS:
cols = setting['columns']
flip_indices = setting['flip_indices']
X, y = load2d(cols=cols)
 X_train, X_val, y_train, y_val = train_test_split(X, y,
test_size=0.2,
random_state=42)
 model4 = load_model(name_transfer_model)
 if freeze:
for layer in model.layers:
layer.trainable = False
model4.layers.pop()
model4.outputs = [model4.layers[-1].output]
model4.layers[-1].outbound_nodes = []
model4.add(Dense(len(cols)))
 model4.compile(loss='mean_squared_error', optimizer="adam")
 flipgen = FlippedImageDataGenerator()
flipgen.flip_indices = setting['flip_indices']
print(X_train.shape)
 print(y_train.shape)
 print(X_val.shape)
 print(y_val.shape)
 hist_final = model4.fit_generator(flipgen.flow(X_train, y_train),
samples_per_epoch=X_train.shape[0],
nb_epoch=epochs,
validation_data=(X_val, y_val))
specialists[cols] = model4
return(specialists)
pd.get_dummies(train_data[categorical[0]] )<categorify>
%%time
specialists2 = fit_specialists(freeze=True,
print_every=10,
epochs=100,
name_transfer_model="my_model2.h5" )
for i in categorical :
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,axis=1 ,inplace=True)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data , dummies],axis=1)
 test_data.drop(i,axis=1,inplace=True )<feature_engineering>
X_test,_ = load2d(test=True)
 def predict_specialist(specialists2,X_test):
y_pred_s = []
for columns, value in specialists2.items() :
smodel = value
y_pred = smodel.predict(X_test)
 y_pred = DataFrame(y_pred,columns=columns)
 y_pred_s.append(y_pred)
 df_y_pred_s = concat(y_pred_s,axis=1)
 return(df_y_pred_s)
 df_y_pred_s = predict_specialist(specialists2,X_test)
 y_pred_s = df_y_pred_s.values
for i in train_data.drop('IsBadBuy',axis=1 ).columns :
if i not in test_data.columns :
test_data[i]=np.zeros(len(test_data)) 
<feature_engineering>
y_pred2 = model2.predict(X_test)
 landmark_nm = read_csv(os.path.expanduser(FTRAIN)).columns[:-1].values
df_y_pred2 = DataFrame(y_pred2,columns = landmark_nm )
for i in test_data.columns :
if i not in train_data.columns :
train_data[i]=np.zeros(len(train_data))<drop_column>
FIdLookup = '.. /input/IdLookupTable.csv'
IdLookup = read_csv(os.path.expanduser(FIdLookup))
def prepare_submission(y_pred2,filename):

 ImageId = IdLookup["ImageId"]
FeatureName = IdLookup["FeatureName"]
RowId = IdLookup["RowId"]
submit = []
for rowId,irow,landmark in zip(RowId,ImageId,FeatureName):
submit.append([rowId,y_pred2[landmark].iloc[irow-1]])
 submit = DataFrame(submit,columns=["RowId","Location"])
 submit["Location"] = submit["Location"]*48 + 48
print(submit.shape)
 if filename == "model2":
submit.to_csv("model2.csv",index=False)
 else:
submit.to_csv("special_model2.csv",index=False)
 prepare_submission(df_y_pred_s,"special_model2")
 prepare_submission(df_y_pred2,"model2" )
test_data=test_data[train_data.drop('IsBadBuy',axis=1 ).columns]<prepare_x_and_y>
df_y_pred_s = df_y_pred_s[df_y_pred3.columns]
df_compare = {}
df_compare["difference"] =(( df_y_pred_s - df_y_pred2)**2 ).mean(axis=1)
 df_compare["RowId"] = range(df_y_pred_s.shape[0])
 df_compare = DataFrame(df_compare)
 df_compare = df_compare.sort_values("difference",ascending=False )
<split><EOS>