Kaggle competition: Mercedes-Benz Greener Manufacturing
Title_Dictionary = {"Capt": "Mr",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Mr",
"Don": "Mr",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Mr",
"Countess": "Royalty",
"Dona": "Mrs",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Officer",
"Lady" : "Royalty"}
full_data['Title'] = full_data.Title.map(Title_Dictionary)
 full_data[['Title', 'Sex', 'Survived']].groupby(['Sex', 'Title'], as_index=True ).mean() 
<categorify>
warnings.filterwarnings("ignore")
 %matplotlib inline
title = pd.get_dummies(full_data.Title)
 title.head()<filter>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 folder = '/kaggle/input/mercedes-benz-greener-manufacturing/'
full_data[full_data.Fare.isnull() ]<groupby>
train_df = pd.read_csv(folder + 'train.csv.zip')
 test_df = pd.read_csv(folder + 'test.csv.zip')
 sub_df = pd.read_csv(folder + 'sample_submission.csv.zip')
 print('train_df: ', train_df.shape)
 print('test_df: ', test_df.shape)
 print('sub_df: ', sub_df.shape )
test_data[["Pclass", "Fare", "Embarked"]].groupby(["Pclass", "Embarked"] ).mean()<create_dataframe>
cat_cols = dfs_train.categoricals.tolist()
fare = pd.DataFrame(data=full_data.Fare.fillna(13.9))
mu_fare = fare.Fare.mean()
sigma_fare =(((fare.Fare-mu_fare)**2 ).mean())**0.5
fare.Fare =(fare.Fare - mu_fare)/ sigma_fare<feature_engineering>
train_df['y'] = np.log(train_df['y'] )
age = pd.DataFrame(data=grouped.Age.apply(lambda x: x.fillna(x.median())))
 mu_a = age.Age.mean()
sigma_a =(((age.Age-mu_a)**2 ).mean())**0.5
age.Age =(age.Age - mu_a)/ sigma_a<categorify>
train_df = train_df[(train_df['y'] > np.percentile(train_df['y'], 0.5)) &(train_df['y'] < np.percentile(train_df['y'], 99.5)) ]
cabin = pd.DataFrame()
cabin['Cabin'] = full_data.Cabin.fillna('U')
 cabin['Cabin'] = cabin.Cabin.map(lambda x: x[0])
 cabin = pd.get_dummies(cabin, drop_first=False)
 cabin.head()<feature_engineering>
y = train_df['y']
train_df.drop(['y'], axis=1, inplace=True )
family = pd.DataFrame()
family['FamilySize'] = full_data['Parch'] + full_data['SibSp'] + 1
family['Family_Single'] = family['FamilySize'].map(lambda s: 1 if s == 1 else 0)
 family['Family_Small'] = family['FamilySize'].map(lambda s: 1 if 2 <= s <= 4 else 0)
 family['Family_Large'] = family['FamilySize'].map(lambda s: 1 if 5 <= s else 0)
 mu_size = family.FamilySize.mean()
sigma_size =(((family.FamilySize-mu_size)**2 ).mean())**0.5
family.FamilySize =(family.FamilySize - mu_size)/ sigma_size
family.head()<concatenate>
class MeanEncoding(BaseEstimator):

 def __init__(self, feature, C=0.1):
self.C = C
self.feature = feature
def fit(self, X_train, y_train):
df = pd.DataFrame({'feature': X_train[self.feature], 'target': y_train} ).dropna()
self.global_mean = df.target.mean()
mean = df.groupby('feature' ).target.mean()
size = df.groupby('feature' ).target.size()
self.encoding =(self.global_mean * self.C + mean * size)/(self.C + size)
 def transform(self, X_test):
X_test[self.feature] = X_test[self.feature].map(self.encoding ).fillna(self.global_mean ).values
return X_test
def fit_transform(self, X_train, y_train):
df = pd.DataFrame({'feature': X_train[self.feature], 'target': y_train} ).dropna()
self.global_mean = df.target.mean()
mean = df.groupby('feature' ).target.mean()
size = df.groupby('feature' ).target.size()
self.encoding =(self.global_mean * self.C + mean * size)/(self.C + size)
 X_train[self.feature] = X_train[self.feature].map(self.encoding ).fillna(self.global_mean ).values
return X_train
features = pd.concat([pclass, sex, embarked, title, fare, age, cabin, family] , axis=1)
 features.head()<split>
for f in cat_cols:
me = MeanEncoding(f, C=0.99)
 me.fit(train_df, y)
 train_df = me.transform(train_df)
 test_df = me.transform(test_df )
labels = train_data.Survived
train_features, val_features, train_labels, val_labels = train_test_split(features[:891],
labels,
test_size = 0.2)
 test_features = features[891:]
train_features.shape, train_labels.shape, val_features.shape, val_labels.shape, test_features.shape<define_variables>
km = KMeans(n_clusters=2, random_state=13)
 km.fit(pd.DataFrame(y))
y_clust = km.predict(pd.DataFrame(y))
m_train = train_features.shape[0]
m_val = val_features.shape[0]
m_test = test_features.shape[0]
n = train_features.shape[1]
print(f" m_train = {m_train} / m_val = {m_val} / m_test = {m_test}, n = {n}" )<define_search_space>
pd.Series(y_clust ).value_counts(normalize=True )
layer_dims = [n, 7, 7, 1]<choose_model_class>
X_train, X_val, y_train, y_val, y_train_clust, y_val_clust = train_test_split(
 train_df, y, pd.Series(y_clust),
test_size=0.25,
stratify=y_clust,
random_state=777
)
k_model = Sequential()
k_model.add(Dense(layer_dims[1], activation='relu',
kernel_regularizer=l2(0.01),
input_dim=n))
k_model.add(Dense(layer_dims[2], activation='relu',
kernel_regularizer=l2(0.01)))
 k_model.add(Dense(layer_dims[3], activation='sigmoid',
kernel_regularizer=l2(0.01)))
 k_model.compile(optimizer=Adam(lr=1e-1),
loss='binary_crossentropy',
metrics=['accuracy'])
 lr_decay = ReduceLROnPlateau(monitor='loss',
patience=1,
verbose=0,
factor=0.5,
min_lr=1e-7)
 k_model.fit(train_features, train_labels, epochs=300,
callbacks=[lr_decay], verbose=0)
 k_train_loss, k_train_acc = k_model.evaluate(train_features, train_labels)
 k_val_loss, k_val_acc = k_model.evaluate(val_features, val_labels)
 print(f'k_model: train accuracy = {round(k_train_acc * 100, 4)}%')
 print(f'k_model: val accuracy = {round(k_val_acc * 100, 4)}%')
 print(f'k_model: val error = {round(( 1 - k_val_acc)* m_val)} examples' )<prepare_x_and_y>
y_train_clust.value_counts(normalize=True )
X_train = train_features.T.values
Y_train = train_labels.T.values.reshape(1, train_labels.shape[0])
 X_val = val_features.T.values
Y_val = val_labels.T.values.reshape(1, val_labels.shape[0])
 X_test = test_features.T.values
X_train.shape, Y_train.shape, X_val.shape, Y_val.shape, X_test.shape<define_variables>
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_sc = pd.DataFrame(scaler.transform(X_train))
X_val_sc = pd.DataFrame(scaler.transform(X_val))
test_df_sc = pd.DataFrame(scaler.transform(test_df))
def random_mini_batches(X, Y, batch_size):

 m_train = X.shape[1]
mini_batches = []
permutation = list(np.random.permutation(m_train))
shuffled_X = X[:, permutation]
shuffled_Y = Y[:, permutation].reshape(( 1,m_train))
num_complete_minibatches = m_train // batch_size
for k in range(0, num_complete_minibatches):
mini_batch_X = shuffled_X[:, k*batch_size:(k+1)*batch_size]
mini_batch_Y = shuffled_Y[:, k*batch_size:(k+1)*batch_size]
mini_batch =(mini_batch_X, mini_batch_Y)
 mini_batches.append(mini_batch)
 if m_train % batch_size != 0:
mini_batch_X = shuffled_X[:, num_complete_minibatches*batch_size:]
mini_batch_Y = shuffled_Y[:, num_complete_minibatches*batch_size:]
mini_batch =(mini_batch_X, mini_batch_Y)
 mini_batches.append(mini_batch)
 return mini_batches<choose_model_class>
pca = PCA(n_components=2)
 pca.fit(X_train_sc)
 train_pca_transformed = pca.transform(X_train_sc )
class Custom_model(object):
def __init__(self, layer_dims):

 self.layer_dims = layer_dims
self.num_layers = len(layer_dims)
 self.parameters = {}
for l in range(1, len(layer_dims)) :
self.parameters[f"W{l}"] = np.random.randn(layer_dims[l], layer_dims[l-1])*0.01
self.parameters[f"b{l}"] = np.zeros(( layer_dims[l], 1))
def get_layer_dims(self):
return self.layer_dims
def get_num_layers(self):
return self.num_layers
def get_params(self, key):
return self.parameters.get(key)
 def set_params(self, key, value):
self.parameters[key] = value
def forward_propagation(self, X):

 caches = []
L = self.get_num_layers() -1
A = X
for l in range(1, L):
A_prev = A
W, b = self.get_params(f'W{l}'), self.get_params(f'b{l}')
 Z = W.dot(A_prev)+ b
A = np.maximum(0,Z)
 assert(A.shape == Z.shape)
 cache =(A_prev, Z)
 caches.append(cache)
 W, b = self.get_params(f'W{L}'), self.get_params(f'b{L}')
 Z = W.dot(A)+ b
AL = 1 /(1 + np.exp(-Z))
assert(AL.shape ==(1,X.shape[1]))
cache =(A, Z)
 caches.append(cache)
 return AL, caches
def compute_cost(self, AL, Y, lambd):

 m = AL.shape[1]
L = self.get_num_layers() -1
assert(Y.shape == AL.shape)
 cross_entropy_cost =(1./m)*(-Y.dot(np.log(AL ).T)-(1-Y ).dot(np.log(1-AL ).T))
L2_reg_cost = 0
if lambd == 0:
pass
else:
for l in range(1, L+1):
L2_reg_cost +=(1./m)*(lambd/2)*(np.sum(np.square(self.get_params(f'W{l}'))))
cost = cross_entropy_cost + L2_reg_cost
cost = np.squeeze(cost)
 return cost
def backward_propagation(self, AL, Y, caches, lambd):

 L = self.get_num_layers() -1
m = AL.shape[1]
assert(Y.shape == AL.shape)
 grads = {}
dAL = -(np.divide(Y, AL)- np.divide(1-Y, 1-AL))
A_prev, Z = caches[L-1]
s = 1./(1.+ np.exp(-Z))
dZ = dAL * s *(1-s)
 W = self.get_params(f"W{L}")
 dW =(1./m)* dZ.dot(A_prev.T)+(lambd/m)* W
db =(1./m)* np.sum(dZ, axis=1, keepdims=True)
 dA_prev = np.dot(W.T,dZ)
 grads[f"dA{L-1}"] = dA_prev
grads[f"dW{L}"], grads[f"db{L}"] = dW, db
for l in reversed(range(1, L)) :
A_prev, Z = caches[l-1]
dZ = np.array(dA_prev, copy=True)
 dZ[Z <= 0] = 0
W = self.get_params(f"W{l}")
 dW =(1./m)* dZ.dot(A_prev.T)+(lambd/m)* W
db =(1./m)* np.sum(dZ, axis=1, keepdims=True)
 dA_prev = np.dot(W.T,dZ)
 grads[f"dA{l-1}"] = dA_prev
grads[f"dW{l}"], grads[f"db{l}"] = dW, db
return grads
def update_parameters_with_gd(self, grads, lr):

 L = self.get_num_layers() -1
for l in range(1, L+1):
self.set_params(f"W{l}", self.get_params(f"W{l}")- lr * grads[f"dW{l}"])
 self.set_params(f"b{l}", self.get_params(f"b{l}")- lr * grads[f"db{l}"])
 def initialize_adam(self):

 v = {}
s = {}
L = self.get_num_layers() -1
for l in range(1, L+1):
v[f"dW{l}"] = np.zeros(( self.get_params(f"W{l}" ).shape))
v[f"db{l}"] = np.zeros(( self.get_params(f"b{l}" ).shape))
s[f"dW{l}"] = np.zeros(( self.get_params(f"W{l}" ).shape))
s[f"db{l}"] = np.zeros(( self.get_params(f"b{l}" ).shape))
return v,s
def update_parameters_with_adam(self, grads, v, s, t, lr, beta1, beta2, epsilon):

 v_corr = {}
s_corr = {}
L = self.get_num_layers() -1
for l in range(1, L+1):
v[f"dW{l}"] = beta1*v[f"dW{l}"] +(1-beta1)*grads[f"dW{l}"]
v[f"db{l}"] = beta1*v[f"db{l}"] +(1-beta1)*grads[f"db{l}"]
v_corr[f"dW{l}"] = v[f"dW{l}"]/(1-beta1**t)
 v_corr[f"db{l}"] = v[f"db{l}"]/(1-beta1**t)
 s[f"dW{l}"] = beta2*s[f"dW{l}"] +(1-beta2)*grads[f"dW{l}"]**2
s[f"db{l}"] = beta2*s[f"db{l}"] +(1-beta2)*grads[f"db{l}"]**2
s_corr[f"dW{l}"] = s[f"dW{l}"]/(1-beta2**t)
 s_corr[f"db{l}"] = s[f"db{l}"]/(1-beta2**t)
 temp_W = v_corr[f"dW{l}"]/(s_corr[f"dW{l}"]**0.5 + epsilon)
 temp_b = v_corr[f"db{l}"]/(s_corr[f"db{l}"]**0.5 + epsilon)
 self.set_params(f"W{l}", self.get_params(f"W{l}")- lr*temp_W)
 self.set_params(f"b{l}", self.get_params(f"b{l}")- lr*temp_b)
 return v, s
def predict(self, X):

 m = X.shape[1]
p = np.zeros(( 1, m))
AL, _ = self.forward_propagation(X)
 for i in range(AL.shape[1]):
if AL[0,i] > 0.5:
p[0,i] = 1
else:
p[0,i] = 0
return np.int64(p)
 def fit(self, X, Y, batch_size, num_epochs, lr, min_lr=1e-5, lambd=0,
beta1=0.9, beta2=0.999, epsilon=1e-8, optimizer='adam', print_cost=False):

 costs = []
lr_0 = lr
t = 0
cost_prev = 1000
if optimizer == "gd":
pass
elif optimizer == "adam":
v, s = self.initialize_adam()
for epoch in range(num_epochs):
minibatches = random_mini_batches(X, Y, batch_size)
 for minibatch in minibatches: 
(minibatch_X, minibatch_Y)= minibatch
AL, caches = self.forward_propagation(minibatch_X)
 cost = self.compute_cost(AL, minibatch_Y, lambd)
 grads = self.backward_propagation(AL, minibatch_Y, caches, lambd)
 t = t + 1
if optimizer == "gd":
self.update_parameters_with_gd(grads, lr)
 elif optimizer == "adam":
v, s = self.update_parameters_with_adam(grads, v, s, t, lr,
beta1, beta2, epsilon)
 if cost > cost_prev and lr > min_lr:
lr = lr / 2
cost_prev = cost
if print_cost and epoch % 20 == 0:
print(f"Cost after epoch {epoch}: {cost}")
 if print_cost and epoch % 1 == 0:
costs.append(cost)
 if print_cost:
plt.plot(np.squeeze(costs))
plt.ylabel("cost")
 plt.xlabel("epochs(x 1)")
 plt.title(f"Learning rate = {lr_0} / {lr}")
 plt.show()<define_variables>
lasso = LassoCV(max_iter=9999)
 lasso.fit(X_train_sc, y_train)
 lasso_train_pred = lasso.predict(X_train_sc)
 lasso_val_pred = lasso.predict(X_val_sc)
 print('train', metrics.r2_score(y_train, lasso_train_pred), 'val', metrics.r2_score(y_val, lasso_val_pred))
records_list = []<train_model>
ridge = RidgeCV()
ridge.fit(X_train_sc, y_train)
 ridge_train_pred = ridge.predict(X_train_sc)
 ridge_val_pred = ridge.predict(X_val_sc)
 print('train', metrics.r2_score(y_train, ridge_train_pred), 'val', metrics.r2_score(y_val, ridge_val_pred))
c_model = Custom_model(layer_dims)
 lr = 1e-1
min_lr = 1e-7
optimizer = 'adam'
batch_size = m_train
num_epochs = 300
lambd = 5e+0
c_model.fit(X_train, Y_train, batch_size=batch_size, num_epochs=num_epochs,
lr=lr, lambd=lambd, optimizer=optimizer, print_cost=True)
 predict_train = c_model.predict(X_train)
 c_train_acc = round(np.sum(predict_train == Y_train)/m_train, 2)
 predict_val = c_model.predict(X_val)
 c_val_acc = round(np.sum(predict_val == Y_val)/m_val, 2)
 print(f"c_model: train accuracy: {c_train_acc * 100}%")
 print(f"c_model: val accuracy = {c_val_acc * 100}%")
 print(f'c_model: val error = {round(( 1 - c_val_acc)* m_val)} examples')
 record = {'layer_dims' : c_model.get_layer_dims() ,
'acc_train' : c_train_acc,
'acc_val' : c_val_acc,
'val_error' : round(( 1 - c_val_acc)* m_val),
'batch_size' : batch_size,
'num_epochs' : num_epochs,
'learning_rate': lr,
'min_lr' : min_lr,
'lambda' : lambd}
records_list.append(record)
 journal = pd.DataFrame(records_list)
 journal<train_model>
enet = ElasticNetCV()
enet.fit(X_train_sc, y_train)
 enet_train_pred = enet.predict(X_train_sc)
 enet_val_pred = enet.predict(X_val_sc)
 print('train', metrics.r2_score(y_train, enet_train_pred), 'val', metrics.r2_score(y_val, enet_val_pred))
c_model_final = Custom_model(layer_dims)
 X = np.concatenate(( X_train, X_val), axis=1)
 Y = np.concatenate(( Y_train, Y_val), axis=1)
 c_model_final.fit(X, Y, batch_size=(m_train + m_val), num_epochs=num_epochs,
lr=lr, min_lr=min_lr, lambd=lambd,
optimizer=optimizer, print_cost=True)
 prediction = c_model_final.predict(X_test)
 submission = pd.DataFrame({"PassengerId": test_data["PassengerId"],
"Survived": np.squeeze(prediction)} ).sort_values(by="PassengerId")
 submission.to_csv("submission.csv", index=False)
 submission.head()<set_options>
huber = HuberRegressor(alpha=0.05)
 huber.fit(X_train_sc, y_train)
 huber_train_pred = huber.predict(X_train_sc)
 huber_val_pred = huber.predict(X_val_sc)
 print('train', metrics.r2_score(y_train, huber_train_pred), 'val', metrics.r2_score(y_val, huber_val_pred))
%matplotlib inline
warnings.filterwarnings("always")
<load_from_csv>
rf = RandomForestRegressor(n_estimators=5)
 rf.fit(X_train_sc, y_train)
 rf_train_pred = rf.predict(X_train_sc)
 rf_val_pred = rf.predict(X_val_sc)
 print('train', metrics.r2_score(y_train, rf_train_pred), 'val', metrics.r2_score(y_val, rf_val_pred))
DATASETS_DIRECTORY = ".. /input"
train_data = pd.read_csv(os.path.join(DATASETS_DIRECTORY, "train.csv"))
test_data = pd.read_csv(os.path.join(DATASETS_DIRECTORY, "test.csv"))
full_data = pd.concat([train_data,test_data])
 targets = train_data["Survived"]
train_data.drop("Survived", axis=1, inplace=True )<categorify>
sub_df['y'] = np.round(np.exp(lasso.predict(test_df_sc)) , 4 )
<categorify><EOS> 
sub_df.to_csv('sub.csv', index=False )