Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
pd.options.display.max_rows=200
pd.set_option('mode.chained_assignment', None)
 simplefilter("ignore", category=ConvergenceWarning)
 simplefilter("ignore", category=RuntimeWarning)
 plt.style.use('fivethirtyeight' )
class Pretrain_CycleGan(keras.Model):
def __init__(
 self,
monet_generator,
photo_generator,
monet_discriminator,
photo_discriminator,
monet_expert,
lambda_cycle=25, 
):
super(Pretrain_CycleGan, self ).__init__()
self.m_gen = monet_generator
self.p_gen = photo_generator
self.m_disc = monet_discriminator
self.p_disc = photo_discriminator
self.m_expert = monet_expert
self.lambda_cycle = lambda_cycle
def compile(
 self,
m_gen_optimizer,
p_gen_optimizer,
m_disc_optimizer,
p_disc_optimizer,
m_expert_optimizer,
gen_loss_fn,
disc_loss_fn,
expert_loss_fn, 
):
super(Pretrain_CycleGan, self ).compile()
self.m_gen_optimizer = m_gen_optimizer
self.p_gen_optimizer = p_gen_optimizer
self.m_disc_optimizer = m_disc_optimizer
self.p_disc_optimizer = p_disc_optimizer
self.m_expert_optimizer = m_expert_optimizer
self.expert_loss_fn = expert_loss
self.gen_loss_fn = gen_loss_fn
self.disc_loss_fn = disc_loss_fn
def train_step(self, batch_data):
real_image, transformed_image, label_is_monet = batch_data
with tf.GradientTape(persistent=True)as tape:
gen_monet = self.m_gen(real_image, training=True)
 gen_photo = self.p_gen(real_image, training=True)
 disc_real_monet = self.m_disc(real_image, training=True)
 disc_real_photo = self.p_disc(real_image, training=True)
 monet_exp_loss=self.expert_loss_fn(self.m_expert(real_image, training=True),label_is_monet)
 monet_gen_loss = self.gen_loss_fn(gen_monet,transformed_image,1)
 photo_gen_loss = self.gen_loss_fn(gen_photo,transformed_image,1)
 monet_disc_loss = self.disc_loss_fn(disc_real_monet, tf.ones_like(disc_real_monet)*label_is_monet)
 photo_disc_loss = self.disc_loss_fn(disc_real_photo, tf.ones_like(disc_real_photo)*(1-label_is_monet))
monet_generator_gradients = tape.gradient(monet_gen_loss,
self.m_gen.trainable_variables)
 photo_generator_gradients = tape.gradient(photo_gen_loss,
self.p_gen.trainable_variables)
 monet_discriminator_gradients = tape.gradient(monet_disc_loss,
self.m_disc.trainable_variables)
 photo_discriminator_gradients = tape.gradient(photo_disc_loss,
self.p_disc.trainable_variables)
 monet_expert_gradients = tape.gradient(monet_exp_loss,
self.m_expert.trainable_variables)
 self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,
self.m_gen.trainable_variables))
self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,
self.p_gen.trainable_variables))
self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,
self.m_disc.trainable_variables))
self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,
self.p_disc.trainable_variables))
self.m_expert_optimizer.apply_gradients(zip(monet_expert_gradients,
self.m_expert.trainable_variables))
return {
"monet_gen_loss": monet_gen_loss,
"photo_gen_loss": photo_gen_loss,
"monet_disc_loss": monet_disc_loss,
"photo_disc_loss": photo_disc_loss,
"monet_exp_loss" : monet_exp_loss
}
def test_step(self, batch_data):
real_image, transformed_image, label_is_monet = batch_data
gen_monet = self.m_gen(real_image, training=True)
 gen_photo = self.p_gen(real_image, training=True)
 disc_real_monet = self.m_disc(real_image, training=True)
 disc_real_photo = self.p_disc(real_image, training=True)
 monet_exp_loss=self.expert_loss_fn(self.m_expert(real_image, training=True),label_is_monet)
 monet_gen_loss = self.gen_loss_fn(gen_monet,transformed_image,1)
 photo_gen_loss = self.gen_loss_fn(gen_photo,transformed_image,1)
 monet_disc_loss = self.disc_loss_fn(disc_real_monet, label_is_monet)
 photo_disc_loss = self.disc_loss_fn(disc_real_photo, 1-label_is_monet)
 return {
"monet_gen_loss": monet_gen_loss,
"photo_gen_loss": photo_gen_loss,
"monet_disc_loss": monet_disc_loss,
"photo_disc_loss": photo_disc_loss,
"monet_exp_loss" : monet_exp_loss
}<compute_test_metric>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
with strategy.scope() :
def discriminator_loss(real, generated):
real_loss = tf.square(tf.ones_like(real)- real)
 generated_loss = tf.square(generated)
 total_disc_loss = real_loss + generated_loss
return total_disc_loss * 0.5 
<compute_test_metric>
train.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True )
with strategy.scope() :
def generator_loss(generated):
return tf.square(tf.ones_like(generated)- generated )<compute_train_metric>
X_train = train.drop('SalePrice', axis=1)
 y_train = np.log1p(train.SalePrice)
 X_test = test
with strategy.scope() :
def calc_cycle_loss(real_image, cycled_image, LAMBDA):
loss1 = tf.reduce_mean(tf.keras.losses.Huber(0.5,reduction=tf.keras.losses.Reduction.NONE )(inception_model(real_image), inception_model(cycled_image)))
 return LAMBDA * loss1<compute_test_metric>
def missing_value_imputation(X):
numerical_features = [feature for feature in X.columns if X[feature].dtype !='O']
ordinal_features = ['LotShape', 'LandSlope', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'Functional', 'FireplaceQu',
'GarageFinish', 'GarageQual', 'GarageCond', 'PoolQC']
categorical_features = [feature for feature in X.columns if feature not in ordinal_features and
feature not in numerical_features]
for feature in numerical_features:
X[feature] = X[feature].fillna(X.groupby('Neighborhood')[feature].transform('median'))
X['MSZoning'].replace({'RL':'R', 'RM':'R', 'RP':'R', 'RM':'R', 'I':'O', 'A':'O', 'C(all)':'O', 'FV':'O'}, inplace=True)
 X['Alley'].replace({np.nan : 'No', 'Grvl':'Yes', 'Pave':'Yes'}, inplace=True)
 X['LandContour'].replace({'Lvl':'Lvl', 'HLS':'Slope', 'Bnk':'Slope', 'Low':'Slope'}, inplace=True)
 X['Condition1'].where(X['Condition1'] == 'Norm', 'Other', inplace=True)
 X['HouseStyle'].where(( X['HouseStyle'] == '1Story')|(X['HouseStyle'] == '2Story')|(X['HouseStyle'] == '1.5Fin'),
'rare', inplace=True)
 X['RoofStyle'].where(( X['RoofStyle'] == 'Gable')|(X['RoofStyle'] == 'Hip'), 'rare', inplace=True)
 X['MasVnrType'].where(X['MasVnrType'] == 'None', 'yes', inplace=True)
 X['MasVnrType'].replace({np.nan : 'None'}, inplace=True)
 X['Exterior1st'].where(( X['Exterior1st'] == 'VinylSd')|(X['Exterior1st'] == 'HdBoard')| 
(X['Exterior1st'] == 'MetalSd')|(X['Exterior1st'] == 'Wd Sdng')| 
(X['Exterior1st'] == 'Plywood'), 'rare', inplace=True)
 X['PavedDrive'].where(X['PavedDrive'] == 'Y', 'N', inplace=True)
 X['Fence'].replace({'MnPrv':'Yes', 'GdPrv':'Yes', 'GdWo':'Yes', 'MnWw':'Yes', np.nan:'No'}, inplace=True)
 X['SaleType'].where(X['SaleType'] == 'WD', 'other', inplace=True)
 X['SaleCondition'].where(( X['SaleCondition'] == 'Normal')|(X['SaleCondition'] == 'Partial')| 
(X['SaleCondition'] == 'Abnorml'), 'other', inplace=True)
 X['Neighborhood'].replace({'Blmngtn':'rare', 'BrDale':'rare', 'Veenker':'rare', 'NPkVill':'rare',
'Blueste':'rare', 'ClearCr':'rare'},
inplace=True)
 for feature in categorical_features:
val = X[feature].value_counts().index[0]
X[feature] = X[feature].fillna(val)
 X['LotShape'].replace({'Reg':1, 'IR1':0, 'IR2':0, 'IR3':0}, inplace=True)
 X['LandSlope'].replace({'Gtl':3, 'Mod':2, 'Sev':1}, inplace=True)
 X['ExterQual'].replace({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['ExterCond'].replace({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['BsmtQual'].replace({np.nan:0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['BsmtCond'].replace({np.nan:0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['BsmtExposure'].replace({np.nan:0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4}, inplace=True)
 X['BsmtFinType1'].replace({np.nan:0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6}, inplace=True)
 X['BsmtFinType2'].replace({np.nan:0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6}, inplace=True)
 X['HeatingQC'].replace({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['KitchenQual'].replace({np.nan:3, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['Functional'].replace({np.nan:8, 'Sal':1, 'Sev':2, 'Maj2':3, 'Maj1':4,
'Mod':5, 'Min2':6, 'Min1':7, 'Typ':8}, inplace=True)
 X['FireplaceQu'].replace({np.nan:0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['GarageFinish'].replace({np.nan:0, 'Unf':1, 'RFn':2, 'Fin':3}, inplace=True)
 X['GarageQual'].replace({np.nan:0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['GarageCond'].replace({np.nan:0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}, inplace=True)
 X['PoolQC'].replace({np.nan:0, 'Fa': 1, 'Gd':2, 'Ex':3}, inplace=True)
 categorical_features = [feature for feature in X.columns if X[feature].dtype == 'O']
for feature in categorical_features:
value_counts = X[feature].value_counts()
for val in value_counts[value_counts / len(X)< 0.015].index:
X[feature].loc[X[feature] == val] = 'rare'
return X
class FeatureEngineering(BaseEstimator, TransformerMixin):
def __init__(self, combined_features=True, drop_underlying_features=True, drop_features=True, correlation_threshold=0.05,
polynomial_features=True, outlier_threshold=1.5, empty_column_dropping_threshold=0.99,
final_correlation_threshold=0.05, test=False):
self.combined_features = combined_features
self.drop_underlying_features = drop_underlying_features
self.drop_features = drop_features
self.correlation_threshold = correlation_threshold
self.polynomial_features = polynomial_features
self.polynomial_features_list = []
self.outlier_threshold = outlier_threshold
self.empty_column_dropping_threshold = empty_column_dropping_threshold
self.low_correlation_drop_list = []
self.one_hot_features_to_drop = []
self.final_low_correlation_drop_list = []
self.final_correlation_threshold = final_correlation_threshold
self.test = test
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
ordinal_features = ['LotShape', 'LandSlope', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'Functional', 'FireplaceQu',
'GarageFinish', 'GarageQual', 'GarageCond', 'PoolQC']
numerical_features = [feature for feature in X.columns if X[feature].dtype !='O' and feature not in ordinal_features]
categorical_features = [feature for feature in X.columns if feature not in ordinal_features and
feature not in numerical_features]
X['GarageYrBlt'] = 2020 - X['GarageYrBlt']
X['YrSold'] = 2020 - X['YrSold']
X['YearBuilt'] = 2020 - X['YearBuilt']
X['YearRemodAdd'] = 2020 - X['YearRemodAdd']
if self.combined_features:
X['OverallGrade'] = X['OverallQual'] * X['OverallCond']
X['ExterGrade'] = X['ExterQual'] * X['ExterCond']
X['BsmtGrade'] = X['BsmtQual'] * X['BsmtCond']
X['BsmtFinType'] = X['BsmtFinType1'] + X['BsmtFinType2']
X['BsmtFinSF'] = X['BsmtFinSF1'] + X['BsmtFinSF2']
X['FlrSF'] = X['1stFlrSF'] + X['2ndFlrSF']
X['BsmtBath'] = X['BsmtFullBath'] + X['BsmtHalfBath']
X['Bath'] = X['FullBath'] + X['HalfBath']
X['GarageGrade'] = X['GarageQual'] * X['GarageCond']
X['Porch'] = X['OpenPorchSF'] + X['EnclosedPorch'] + X['3SsnPorch'] + X['ScreenPorch']
numerical_features.extend(['OverallGrade', 'ExterGrade', 'BsmtGrade', 'BsmtFinType', 'BsmtFinSF', 'FlrSF',
'BsmtBath', 'Bath', 'GarageGrade', 'Porch'])
 if self.combined_features and self.drop_underlying_features:
to_drop = [ 'BsmtFinType1','BsmtFinType2', 'BsmtFinSF1', '1stFlrSF', 'BsmtFullBath',
'BsmtHalfBath', 'FullBath', 'HalfBath', 'GarageCond',
'OpenPorchSF', 'BsmtFinSF2', '2ndFlrSF', 'EnclosedPorch',
'3SsnPorch', 'ScreenPorch']
X.drop(to_drop, axis=1, inplace=True)
 if self.drop_features:
to_drop = ['Utilities', 'PoolQC', 'MiscFeature', 'Street',
'Condition2', 'MasVnrType', 'LowQualFinSF', 'Alley', 'MiscVal', 'Fence',
'KitchenAbvGr', 'PoolArea', 'Street', 'RoofMatl', 'Exterior2nd', 'Heating']
X.drop(to_drop, axis=1, inplace=True)
 ordinal_features = [feature for feature in X.columns if X[feature].dtype != 'O' and feature not in numerical_features]
numerical_features = [feature for feature in X.columns if X[feature].dtype != 'O' and feature not in ordinal_features]
categorical_features = [feature for feature in X.columns if X[feature].dtype == 'O']
if self.polynomial_features:
if len(X)> 800 and not self.test:
y = train['SalePrice'].iloc[X.index].to_frame()
saleprice = pd.concat([X, y], axis=1 ).corr() ['SalePrice']
self.polynomial_features_list = list(saleprice.sort_values(ascending=False ).index[1:self.polynomial_features + 2])
 for feature in self.polynomial_features_list:
for i in [2, 3]:
X[f'{feature} - sq{i}'] = X[feature] ** i
else:
for feature in self.polynomial_features_list:
for i in [2, 3]:
X[f'{feature} - sq{i}'] = X[feature] ** i
if self.outlier_threshold:
numerical_features = [feature for feature in X.columns
if feature not in ordinal_features and X[feature].dtype != 'O' ]
for feature in numerical_features:
unique_vals = X[feature].nunique()
if unique_vals > 10:
q1 = np.percentile(X[feature], 25)
 q3 = np.percentile(X[feature], 75)
 iqr = q3 - q1
if not iqr:
iqr = 1
cut_off = iqr * self.outlier_threshold
lower, upper = q1 - cut_off, q3 + cut_off
X[feature].where(~(X[feature] > upper), upper, inplace=True)
 X[feature].where(~(X[feature] < lower), lower, inplace=True)
 if self.correlation_threshold:
if len(X)> 800 and not self.test:
self.low_correlation_drop_list = []
y = train['SalePrice'].iloc[X.index].to_frame()
saleprice = pd.concat([X, y], axis=1 ).corr() ['SalePrice']
corr_dict = saleprice.sort_values(ascending=False ).to_frame().to_dict() ['SalePrice']
for key, value in corr_dict.items() :
if value < self.correlation_threshold and value > - self.correlation_threshold:
self.low_correlation_drop_list.append(key)
 X = X.drop(self.low_correlation_drop_list, axis=1)
 else:
X = X.drop(self.low_correlation_drop_list, axis=1)
 numerical_features = [feature for feature in X.columns if X[feature].dtype != 'O']
categorical_features = [feature for feature in X.columns if feature not in numerical_features]
X.replace({0 : 1e-5}, inplace=True)
 X_index = X.index
num_pipeline = Pipeline([('scale', MinMaxScaler(feature_range=(1e-5, 1))), 
('power_transform', PowerTransformer(method='box-cox')) ])
 transformer = ColumnTransformer([('num_pipeline', num_pipeline, numerical_features)])
 numerical_features_transformed = transformer.fit_transform(X)
 numerical_df = pd.DataFrame(numerical_features_transformed, columns=numerical_features, index=X.index)
 X = pd.concat([numerical_df, X.loc[:, categorical_features]], axis=1)
 if not self.test:
X_total = pd.concat([X, X_train_imputed.loc[:, categorical_features].loc[~(X_train_imputed.index.isin(X.index)) ]])
 X_total = pd.get_dummies(X_total)
 one_hot_features = [feature for feature in X_total.columns if feature not in numerical_features]
else:
X = pd.get_dummies(X)
 if self.empty_column_dropping_threshold:
if len(X)> 800 and not self.test:
self.one_hot_features_to_drop = []
for feature in one_hot_features:
zero_count = X_total[feature].value_counts() [0] / len(X_total)
 if zero_count > self.empty_column_dropping_threshold:
self.one_hot_features_to_drop.append(feature)
 X_total.drop(feature, axis=1, inplace=True)
 elif not self.test:
X_total.drop(self.one_hot_features_to_drop, axis=1, inplace=True)
 else:
X.drop(self.one_hot_features_to_drop, axis=1, inplace=True)
 if not self.test:
X = X_total.dropna()
if self.final_correlation_threshold:
if len(X)> 800 and not self.test:
self.final_low_correlation_drop_list = []
y = train['SalePrice'].iloc[X.index].to_frame()
saleprice = pd.concat([X, y], axis=1 ).corr() ['SalePrice']
corr_dict = saleprice.sort_values(ascending=False ).to_frame().to_dict() ['SalePrice']
for key, value in corr_dict.items() :
if value < self.final_correlation_threshold and value > - self.final_correlation_threshold:
self.final_low_correlation_drop_list.append(key)
 X = X.drop(self.final_low_correlation_drop_list, axis=1)
 else:
X = X.drop(self.final_low_correlation_drop_list, axis=1)
 return X
with strategy.scope() :
def identity_loss(real_image, translated_image, LAMBDA):
loss = tf.reduce_mean(tf.keras.losses.Huber(0.5,reduction=tf.keras.losses.Reduction.NONE )(tf.nn.avg_pool2d(real_image, ksize=32, strides=16, padding="VALID"), tf.nn.avg_pool2d(translated_image, ksize=32, strides=16, padding="VALID")))
 return LAMBDA * loss<compute_test_metric>
X_train_imputed = missing_value_imputation(X_train.copy() )
with strategy.scope() :
def pretrain_identity_loss(real_image, translated_image, LAMBDA):
loss = tf.reduce_mean(tf.abs(real_image - translated_image))
return LAMBDA * loss<compute_train_metric>
param_grid = {'feature_engineering__combined_features':[True],
'feature_engineering__drop_features':[True],
'feature_engineering__drop_underlying_features':[True],
'feature_engineering__polynomial_features':[9],
'feature_engineering__correlation_threshold': [0.07],
'feature_engineering__outlier_threshold':[1.5],
'feature_engineering__empty_column_dropping_threshold':[0.99],
'feature_engineering__final_correlation_threshold':[False]
}
with strategy.scope() :
def pretrain_disc_loss(image,label):
return tf.square(label - image)
 def expert_loss(generated,label):
return tf.keras.losses.BinaryCrossentropy(from_logits=True,reduction=tf.keras.losses.Reduction.SUM )(tf.reduce_mean(label,axis=[1,2,3]),tf.reduce_mean(generated,axis=[1,2,3]))<choose_model_class>
linreg = Pipeline([('feature_engineering', FeatureEngineering()), 
('linreg', Ridge())])
 grid = GridSearchCV(linreg, param_grid, cv=5, scoring='neg_mean_squared_error', verbose=2, n_jobs=-1)
 grid.fit(X_train_imputed.copy() , y_train )
with strategy.scope() :
monet_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 monet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 monet_expert_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 pt_monet_generator = Generator()
pt_photo_generator = Generator()
pt_monet_discriminator = Discriminator()
pt_photo_discriminator = Discriminator()
pt_monet_expert = Discriminator()
monet_expert = Discriminator()<choose_model_class>
grid.best_params_
with strategy.scope() :
cycle_gan_model = CycleGan(
 monet_generator, photo_generator, monet_discriminator, photo_discriminator,monet_expert 
)
 cycle_gan_model.compile(
 m_gen_optimizer = monet_generator_optimizer,
p_gen_optimizer = photo_generator_optimizer,
m_disc_optimizer = monet_discriminator_optimizer,
p_disc_optimizer = photo_discriminator_optimizer,
gen_loss_fn = generator_loss,
disc_loss_fn = discriminator_loss,
cycle_loss_fn = calc_cycle_loss,
identity_loss_fn = identity_loss,
expert_loss_fn = expert_loss,
)<choose_model_class>
fe = FeatureEngineering(**{'combined_features': True,
'correlation_threshold': 0.07,
'drop_features': True,
'drop_underlying_features': True,
'empty_column_dropping_threshold': 0.99,
'final_correlation_threshold':False,
'outlier_threshold': 1.5,
'polynomial_features': 9} )
with strategy.scope() :
pretrain_cycle_gan_model = Pretrain_CycleGan(pt_monet_generator, pt_photo_generator, pt_monet_discriminator, pt_photo_discriminator,pt_monet_expert)
 pretrain_cycle_gan_model.compile(
 m_gen_optimizer = monet_generator_optimizer,
p_gen_optimizer = photo_generator_optimizer,
m_disc_optimizer = monet_discriminator_optimizer,
p_disc_optimizer = photo_discriminator_optimizer,
m_expert_optimizer = monet_expert_optimizer,
gen_loss_fn = pretrain_identity_loss,
disc_loss_fn = pretrain_disc_loss,
expert_loss_fn = expert_loss,
)<train_model>
X_train_imputed = missing_value_imputation(X_train.copy())
 X_test_imputed = missing_value_imputation(X_test.copy() )
hist=pretrain_cycle_gan_model.fit(pretrain_ds,validation_data=val_pretrain_ds, epochs=33 ).history<load_pretrained>
X_train_fe = fe.transform(X_train_imputed.copy() )
pretrain_cycle_gan_model.save_weights('premonet.h5' )<load_pretrained>
fe.test = True
cycle_gan_model.built = True
cycle_gan_model.load_weights('premonet.h5' )<init_hyperparams>
X_test_fe = fe.transform(X_test_imputed.copy() )
! rm premonet.h5
disc_m_loss1=[]
disc_p_loss1=[]<train_model>
fe.test = False
%%time
fids=[]
best_fid=999999999
for epoch in range(1,48):
print("Epoch = ",epoch)
 hist=cycle_gan_model.fit(gan_ds,steps_per_epoch=1500, epochs=1 ).history
if epoch>35:
cur_fid=FID(fast_photo_ds,monet_generator)
 fids.append(cur_fid)
 print("After epoch
".format(epoch,cur_fid))
if epoch>42:
best_fid=cur_fid
monet_generator.save('monet_generator_'+str(epoch)+'.h5')
<categorify>
corr_sale = pd.concat([X_train_fe, y_train], axis=1 ).corr() ['SalePrice']
_, ax = plt.subplots(5, 3, figsize=(32, 32))
for i, img in enumerate(photo_ds.batch(1 ).take(5)) :
prediction = monet_generator(img, training=False)
 cycledphoto = photo_generator(prediction, training=False)
 prediction =(prediction * 127.5 + 127.5)[0].numpy().astype(np.uint8)
 cycledphoto =(cycledphoto * 127.5 + 127.5)[0].numpy().astype(np.uint8)
 img =(img[0] * 127.5 + 127.5 ).numpy().astype(np.uint8)
 ax[i, 0].imshow(img)
 ax[i, 1].imshow(prediction)
 ax[i, 2].imshow(cycledphoto)
 ax[i, 0].set_title("Input Photo")
 ax[i, 1].set_title("Monet-esque")
 ax[i, 2].set_title("Cycled Photo")
 ax[i, 0].axis("off")
 ax[i, 1].axis("off")
 ax[i, 2].axis("off")
 plt.show()<data_type_conversions>
corr_sale.sort_values(ascending=False )
_, ax = plt.subplots(5, 3, figsize=(32, 32))
for i, img in enumerate(monet_ds.batch(1 ).take(5)) :
prediction = photo_generator(img, training=False)
 cycledphoto = monet_generator(prediction, training=False)
 prediction =(prediction * 127.5 + 127.5)[0].numpy().astype(np.uint8)
 cycledphoto =(cycledphoto * 127.5 + 127.5)[0].numpy().astype(np.uint8)
 img =(img[0] * 127.5 + 127.5 ).numpy().astype(np.uint8)
 ax[i, 0].imshow(img)
 ax[i, 1].imshow(prediction)
 ax[i, 2].imshow(cycledphoto)
 ax[i, 0].set_title("Input Monet")
 ax[i, 1].set_title("Generated Photo")
 ax[i, 2].set_title("Cycled Monet")
 ax[i, 0].axis("off")
 ax[i, 1].axis("off")
 ax[i, 2].axis("off")
 plt.show()<data_type_conversions>
def results(cv_results_, n):
df = pd.DataFrame(cv_results_)[['params', 'mean_test_score']].nlargest(n, columns='mean_test_score')
 for i in range(len(df)) :
print(f'{df.iloc[i, 0]} : {df.iloc[i, 1]}' )
%%time
i = 1
for img in fast_photo_ds:
prediction = monet_generator(img, training=False ).numpy()
prediction =(prediction * 127.5 + 127.5 ).astype(np.uint8)
 for pred in prediction:
im = PIL.Image.fromarray(pred)
 im.save(".. /images/" + str(i)+ ".jpg")
 i += 1 
<load_pretrained>
def parameter_plot(model, X, y, n_estimators=[100, 200, 300, 400, 600, 900, 1300], hyper_param=None, **kwargs):
param_name, param_vals = hyper_param
param_grid = {'n_estimators':n_estimators,
f'{param_name}':param_vals}
grid = GridSearchCV(model(**kwargs), param_grid,
cv=RepeatedKFold(n_splits=8, n_repeats=1, random_state=42),
scoring='neg_root_mean_squared_error', n_jobs=-1, verbose=2)
 grid.fit(X, y)
 results = pd.DataFrame(grid.cv_results_)['mean_test_score'].values
results = results.reshape(len(param_vals), len(n_estimators))
plt.figure(figsize=(15, 9))
for i in range(1, len(param_vals)+ 1):
plt.plot(n_estimators, results[i-1], label=f'{param_name} - {param_vals[i-1]}')
 plt.legend()
plt.show()
shutil.make_archive("/kaggle/working/images", 'zip', "/kaggle/images" )<set_options>
def learning_curve_plotter(Model, X, y, params_1, params_2, step=50):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
 fig, ax = plt.subplots(1, 2, sharey=True, figsize=(16, 7))
for i,(name, params)in enumerate([params_1, params_2]):
train_score = []
val_score = []
for j in range(100, len(X_train), step):
model = Model(**params ).fit(X_train[:j], y_train[:j])
 y_train_preds = model.predict(X_train[:j])
 y_test_preds = model.predict(X_test)
 train_score.append(np.sqrt(mean_squared_error(y_train[:j], y_train_preds)))
 val_score.append(np.sqrt(mean_squared_error(y_test, y_test_preds)))
 ax[i].plot(train_score, 'r-', label='Training error')
 ax[i].plot(val_score, 'b-', label='Validation error')
 ax[i].set_title(f'{name}', fontsize=18)
 ax[i].set_xlabel('Training set size', fontsize=15, labelpad=10)
 ax[i].set_ylabel('Root mean squared log error', fontsize=15, labelpad=10)
 ax[i].legend()
plt.show()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Device:', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE
print(tf.__version__ )<define_variables>
param_grid_ridge = {'alpha':0.1 * np.arange(1, 70)}
GCS_PATH_MONET = KaggleDatasets().get_gcs_path('monet-tfrecords-extdata')
 GCS_PATH = KaggleDatasets().get_gcs_path('gan-getting-started')
 GCS_PATH_MONET, GCS_PATH<define_variables>
grid_ridge = GridSearchCV(Ridge() , param_grid_ridge,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
MONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))
print('Monet TFRecord Files:', len(MONET_FILENAMES))
PHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))
print('Photo TFRecord Files:', len(PHOTO_FILENAMES))
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 n_monet_samples = count_data_items(MONET_FILENAMES)
 n_photo_samples = count_data_items(PHOTO_FILENAMES)
 BATCH_SIZE = 4
EPOCHS_NUM = 30
print(f'Monet TFRecord files: {len(MONET_FILENAMES)}')
 print(f'Monet image files: {n_monet_samples}')
 print(f'Photo TFRecord files: {len(PHOTO_FILENAMES)}')
 print(f'Photo image files: {n_photo_samples}')
 print(f"Batch_size: {BATCH_SIZE}")
 print(f"Epochs number: {EPOCHS_NUM}" )<categorify>
grid_ridge.fit(X_train_fe, y_train )
IMAGE_SIZE = [256, 256]
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image =(tf.cast(image, tf.float32)/ 127.5)- 1
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_tfrecord(example):
tfrecord_format = {
"image_name": tf.io.FixedLenFeature([], tf.string),
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 return image<normalization>
best_params_ridge = {'alpha': 3.8000000000000003}
def data_augment(image):
p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_crop >.5:
image = tf.image.resize(image, [286, 286])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_crop >.9:
image = tf.image.resize(image, [300, 300])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_rotate >.9:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.7:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=1)
 if p_spatial >.6:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.9:
image = tf.image.transpose(image)
 return image
def load_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_gan_dataset(monet_files, photo_files, augment=None, repeat=True, shuffle=True, batch_size=1):
monet_ds = load_dataset(monet_files)
 photo_ds = load_dataset(photo_files)
 if augment:
monet_ds = monet_ds.map(augment, num_parallel_calls=AUTO)
 photo_ds = photo_ds.map(augment, num_parallel_calls=AUTO)
 if repeat:
monet_ds = monet_ds.repeat()
photo_ds = photo_ds.repeat()
if shuffle:
monet_ds = monet_ds.shuffle(2048)
 photo_ds = photo_ds.shuffle(2048)
 monet_ds = monet_ds.batch(batch_size, drop_remainder=True)
 photo_ds = photo_ds.batch(batch_size, drop_remainder=True)
 monet_ds = monet_ds.cache()
photo_ds = photo_ds.cache()
monet_ds = monet_ds.prefetch(AUTO)
 photo_ds = photo_ds.prefetch(AUTO)
 gan_ds = tf.data.Dataset.zip(( monet_ds, photo_ds))
return gan_ds<create_dataframe>
param_grid_lasso = {'alpha': 0.0001 * np.arange(1, 100)}
full_dataset = get_gan_dataset(MONET_FILENAMES, PHOTO_FILENAMES, augment=data_augment, repeat=True, shuffle=True, batch_size=BATCH_SIZE)
<define_variables>
grid_lasso = GridSearchCV(Lasso() , param_grid_lasso,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
example_monet , example_photo = next(iter(full_dataset))<define_search_model>
grid_lasso.fit(X_train_fe, y_train )
OUTPUT_CHANNELS = 3
def downsample(filters, size, apply_instancenorm=True):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2D(filters, size, strides=2, padding='same',
kernel_initializer=initializer, use_bias=False))
if apply_instancenorm:
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
result.add(layers.LeakyReLU())
 return result<define_search_model>
best_params_lasso = {'alpha': 0.0002}
def upsample(filters, size, apply_dropout=False):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2DTranspose(filters, size, strides=2,
padding='same',
kernel_initializer=initializer,
use_bias=False))
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
if apply_dropout:
result.add(layers.Dropout(0.5))
result.add(layers.ReLU())
 return result<define_search_space>
param_grid_elastic = {'alpha': [0.0001, 0.001, 0.01, 0.1, 0.5, 1, 5, 10],
'l1_ratio':0.01 * np.arange(10)}
def Generator() :
inputs = layers.Input(shape=[256,256,3])
 down_stack = [
downsample(64, 4, apply_instancenorm=False),
downsample(128, 4),
downsample(256, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
]
up_stack = [
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4),
upsample(256, 4),
upsample(128, 4),
upsample(64, 4),
]
initializer = tf.random_normal_initializer(0., 0.02)
 last = layers.Conv2DTranspose(OUTPUT_CHANNELS, 4,
strides=2,
padding='same',
kernel_initializer=initializer,
activation='tanh')
 x = inputs
skips = []
for down in down_stack:
x = down(x)
 skips.append(x)
 skips = reversed(skips[:-1])
 for up, skip in zip(up_stack, skips):
x = up(x)
 x = layers.Concatenate()([x, skip])
 x = last(x)
 return keras.Model(inputs=inputs, outputs=x )<define_search_model>
grid_elastic = GridSearchCV(ElasticNet() , param_grid_elastic,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
def Discriminator() :
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 inp = layers.Input(shape=[256, 256, 3], name='input_image')
 x = inp
down1 = downsample(64, 4, False )(x)
 down2 = downsample(128, 4 )(down1)
 down3 = downsample(256, 4 )(down2)
 zero_pad1 = layers.ZeroPadding2D()(down3)
 conv = layers.Conv2D(512, 4, strides=1,
kernel_initializer=initializer,
use_bias=False )(zero_pad1)
 norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init )(conv)
 leaky_relu = layers.LeakyReLU()(norm1)
 zero_pad2 = layers.ZeroPadding2D()(leaky_relu)
 last = layers.Conv2D(1, 4, strides=1,
kernel_initializer=initializer )(zero_pad2)
 return tf.keras.Model(inputs=inp, outputs=last )<define_search_model>
grid_elastic.fit(X_train_fe, y_train )
with strategy.scope() :
monet_generator = Generator()
photo_generator = Generator()
monet_discriminator = Discriminator()
photo_discriminator = Discriminator() 
<choose_model_class>
best_params_elastic = {'alpha': 0.001, 'l1_ratio': 0.09}
class CycleGan(keras.Model):
def __init__(
 self,
monet_generator,
photo_generator,
monet_discriminator,
photo_discriminator,
lambda_cycle=10, 
):
super(CycleGan, self ).__init__()
self.m_gen = monet_generator
self.p_gen = photo_generator
self.m_disc = monet_discriminator
self.p_disc = photo_discriminator
self.lambda_cycle = lambda_cycle
def compile(
 self,
m_gen_optimizer,
p_gen_optimizer,
m_disc_optimizer,
p_disc_optimizer,
gen_loss_fn,
disc_loss_fn,
cycle_loss_fn,
identity_loss_fn 
):
super(CycleGan, self ).compile()
self.m_gen_optimizer = m_gen_optimizer
self.p_gen_optimizer = p_gen_optimizer
self.m_disc_optimizer = m_disc_optimizer
self.p_disc_optimizer = p_disc_optimizer
self.gen_loss_fn = gen_loss_fn
self.disc_loss_fn = disc_loss_fn
self.cycle_loss_fn = cycle_loss_fn
self.identity_loss_fn = identity_loss_fn
def train_step(self, batch_data):
real_monet, real_photo = batch_data
with tf.GradientTape(persistent=True)as tape:
fake_monet = self.m_gen(real_photo, training=True)
 cycled_photo = self.p_gen(fake_monet, training=True)
 fake_photo = self.p_gen(real_monet, training=True)
 cycled_monet = self.m_gen(fake_photo, training=True)
 same_monet = self.m_gen(real_monet, training=True)
 same_photo = self.p_gen(real_photo, training=True)
 disc_real_monet = self.m_disc(real_monet, training=True)
 disc_real_photo = self.p_disc(real_photo, training=True)
 disc_fake_monet = self.m_disc(fake_monet, training=True)
 disc_fake_photo = self.p_disc(fake_photo, training=True)
 monet_gen_loss = self.gen_loss_fn(disc_fake_monet)
 photo_gen_loss = self.gen_loss_fn(disc_fake_photo)
 total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle)+ self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)
 total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle)
 total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)
 monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)
 photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)
 monet_generator_gradients = tape.gradient(total_monet_gen_loss,
self.m_gen.trainable_variables)
 photo_generator_gradients = tape.gradient(total_photo_gen_loss,
self.p_gen.trainable_variables)
 monet_discriminator_gradients = tape.gradient(monet_disc_loss,
self.m_disc.trainable_variables)
 photo_discriminator_gradients = tape.gradient(photo_disc_loss,
self.p_disc.trainable_variables)
 self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,
self.m_gen.trainable_variables))
self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,
self.p_gen.trainable_variables))
self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,
self.m_disc.trainable_variables))
self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,
self.p_disc.trainable_variables))
return {
"monet_gen_loss": total_monet_gen_loss,
"photo_gen_loss": total_photo_gen_loss,
"monet_disc_loss": monet_disc_loss,
"photo_disc_loss": photo_disc_loss
} 
<compute_train_metric>
param_grid_svr = {'kernel': ['rbf'],
'degree':[1],
'epsilon':[0.001, 0.01, 0.008, 0.013],
'C':[0.1, 0.5, 1, 20, 25],
'gamma':[0.0004, 0.0006, 0.0007, 0.0008]}
with strategy.scope() :
def discriminator_loss(real, generated):
real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE )(tf.ones_like(real), real)
 generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE )(tf.zeros_like(generated), generated)
 total_disc_loss = real_loss + generated_loss
return total_disc_loss * 0.5 
<compute_train_metric>
grid_svr = GridSearchCV(SVR() , param_grid_svr,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
with strategy.scope() :
def generator_loss(generated):
return tf.keras.losses.BinaryCrossentropy(from_logits=True,
reduction=tf.keras.losses.Reduction.NONE )(tf.ones_like(generated), generated )<compute_test_metric>
grid_svr.fit(X_train_fe, y_train )
with strategy.scope() :
def calc_cycle_loss(real_image, cycled_image, LAMBDA):
loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))
return LAMBDA * loss1<compute_test_metric>
best_params_svr = {'C': 25, 'degree': 1, 'epsilon': 0.008, 'gamma': 0.0004, 'kernel': 'rbf'}
with strategy.scope() :
def identity_loss(real_image, same_image, LAMBDA):
loss = tf.reduce_mean(tf.abs(real_image - same_image))
return LAMBDA * 0.5 * loss<choose_model_class>
param_grid_extra = {'max_depth': [8, 11],
'max_samples':[0.6, 0.8, None],
'max_features':[0.5, 0.8, None],
'n_estimators': [200, 600, 1000]}
with strategy.scope() :
monet_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 monet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5 )<choose_model_class>
grid_extra = GridSearchCV(ExtraTreesRegressor() , param_grid_extra,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
with strategy.scope() :
cycle_gan_model = CycleGan(
 monet_generator, photo_generator, monet_discriminator, photo_discriminator 
)
 cycle_gan_model.compile(
 m_gen_optimizer = monet_generator_optimizer,
p_gen_optimizer = photo_generator_optimizer,
m_disc_optimizer = monet_discriminator_optimizer,
p_disc_optimizer = photo_discriminator_optimizer,
gen_loss_fn = generator_loss,
disc_loss_fn = discriminator_loss,
cycle_loss_fn = calc_cycle_loss,
identity_loss_fn = identity_loss 
)
<train_model>
grid_extra.fit(X_train_fe, y_train )
cycle_gan_model.fit(
 full_dataset,
epochs=EPOCHS_NUM,
steps_per_epoch=(max(n_monet_samples, n_photo_samples)//BATCH_SIZE), 
)
<predict_on_test>
best_params_extra = {'max_depth': 8, 'max_features': 0.5, 'max_samples': 0.8, 'n_estimators': 200}
def predict_and_save(input_ds, generator_model, output_path):
i = 1
for img in input_ds:
prediction = generator_model(img, training=False)[0].numpy()
prediction =(prediction * 127.5 + 127.5 ).astype(np.uint8)
 im = PIL.Image.fromarray(prediction)
 im.save(f'{output_path}{str(i)}.jpg')
 i += 1<define_variables>
param_grid_random = {'max_depth': [8, 11],
'max_samples':[0.6, 0.8, None],
'max_features':[0.5, 0.7, None],
'n_estimators': [200, 500, 1000]}
os.makedirs('.. /images/')
 predict_and_save(load_dataset(PHOTO_FILENAMES ).batch(1), monet_generator, '.. /images/')
<load_pretrained>
grid_random = GridSearchCV(RandomForestRegressor() , param_grid_random,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
shutil.make_archive('/kaggle/working/images/', 'zip', '.. /images')
 print(f"Generated samples: {len([name for name in os.listdir('.. /images/')if os.path.isfile(os.path.join('.. /images/', name)) ])}" )<set_options>
grid_random.fit(X_train_fe, y_train )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Device:', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE
print(tf.__version__ )<define_variables>
best_params_random = {'max_depth': 8, 'max_features': 0.5, 'max_samples': 0.8, 'n_estimators': 1000}
GCS_PATH_MONET = KaggleDatasets().get_gcs_path('monet-tfrecords-extdata')
 GCS_PATH = KaggleDatasets().get_gcs_path('gan-getting-started')
 GCS_PATH_MONET, GCS_PATH<define_variables>
param_grid_gradient = {'n_estimators':[250, 400, 600],
'learning_rate':[0.035, 0.05],
'subsample':[0.5, 0.7],
'max_depth':[3],
'max_features':[0.5, 0.7]
}
MONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))
print('Monet TFRecord Files:', len(MONET_FILENAMES))
PHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))
print('Photo TFRecord Files:', len(PHOTO_FILENAMES))
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 n_monet_samples = count_data_items(MONET_FILENAMES)
 n_photo_samples = count_data_items(PHOTO_FILENAMES)
 BATCH_SIZE = 4
EPOCHS_NUM = 30
print(f'Monet TFRecord files: {len(MONET_FILENAMES)}')
 print(f'Monet image files: {n_monet_samples}')
 print(f'Photo TFRecord files: {len(PHOTO_FILENAMES)}')
 print(f'Photo image files: {n_photo_samples}')
 print(f"Batch_size: {BATCH_SIZE}")
 print(f"Epochs number: {EPOCHS_NUM}" )<categorify>
grid_gradient = GridSearchCV(GradientBoostingRegressor() , param_grid_gradient,
cv=RepeatedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='neg_root_mean_squared_error', verbose=2, n_jobs=-1 )
IMAGE_SIZE = [256, 256]
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image =(tf.cast(image, tf.float32)/ 127.5)- 1
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_tfrecord(example):
tfrecord_format = {
"image_name": tf.io.FixedLenFeature([], tf.string),
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 return image<normalization>
grid_gradient.fit(X_train_fe, y_train )
def data_augment(image):
p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_crop >.5:
image = tf.image.resize(image, [286, 286])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_crop >.9:
image = tf.image.resize(image, [300, 300])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_rotate >.9:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.7:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=1)
 if p_spatial >.6:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.9:
image = tf.image.transpose(image)
 return image
def load_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_gan_dataset(monet_files, photo_files, augment=None, repeat=True, shuffle=True, batch_size=1):
monet_ds = load_dataset(monet_files)
 photo_ds = load_dataset(photo_files)
 if augment:
monet_ds = monet_ds.map(augment, num_parallel_calls=AUTO)
 photo_ds = photo_ds.map(augment, num_parallel_calls=AUTO)
 if repeat:
monet_ds = monet_ds.repeat()
photo_ds = photo_ds.repeat()
if shuffle:
monet_ds = monet_ds.shuffle(2048)
 photo_ds = photo_ds.shuffle(2048)
 monet_ds = monet_ds.batch(batch_size, drop_remainder=True)
 photo_ds = photo_ds.batch(batch_size, drop_remainder=True)
 monet_ds = monet_ds.cache()
photo_ds = photo_ds.cache()
monet_ds = monet_ds.prefetch(AUTO)
 photo_ds = photo_ds.prefetch(AUTO)
 gan_ds = tf.data.Dataset.zip(( monet_ds, photo_ds))
return gan_ds<create_dataframe>
best_params_gradient = {'learning_rate': 0.035, 'max_depth': 3, 'max_features': 0.5,
'n_estimators': 400, 'subsample': 0.7}
full_dataset = get_gan_dataset(MONET_FILENAMES, PHOTO_FILENAMES, augment=data_augment, repeat=True, shuffle=True, batch_size=BATCH_SIZE )<define_variables>
def objective(trial, X, y):
params = {'n_estimators':2000,
'learning_rate':trial.suggest_uniform('learning_rate', 0.005, 0.01),
'subsample':trial.suggest_categorical('subsample', [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),
'max_depth':trial.suggest_int('max_depth', 3, 11),
'colsample_bylevel':trial.suggest_categorical('colsample_bylevel', [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),
'reg_lambda':trial.suggest_loguniform('reg_lambda', 1e-3, 100),
'reg_alpha':trial.suggest_loguniform('reg_alpha', 1e-3, 100),
'n_jobs':-1}
model = XGBRegressor(**params)
 split = KFold(n_splits=5)
 train_scores = []
test_scores = []
for train_idx, val_idx in split.split(X_train_fe):
X_tr = X_train_fe.iloc[train_idx]
X_val = X_train_fe.iloc[val_idx]
y_tr = y_train.iloc[train_idx]
y_val = y_train.iloc[val_idx]
model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=30, verbose=0,
callbacks=[optuna.integration.XGBoostPruningCallback(trial, observation_key="validation_0-rmse")] 
)
 train_score = np.round(np.sqrt(mean_squared_error(y_tr, model.predict(X_tr))), 4)
 test_score = np.round(np.sqrt(mean_squared_error(y_val, model.predict(X_val))), 4)
 train_scores.append(train_score)
 test_scores.append(test_score)
 print(f'train score : {train_scores}')
 print(f'test score : {test_scores}')
 train_score = np.round(np.mean(train_scores), 4)
 test_score = np.round(np.mean(test_scores), 4)
 print(f'TRAIN RMSE : {train_score} || TEST RMSE : {test_score}')
 return test_score
example_monet , example_photo = next(iter(full_dataset))<define_search_model>
optimize = partial(objective, X=X_train_fe, y=y_train)
 study = optuna.create_study(direction='minimize')
 study.optimize(optimize, n_trials=100)
OUTPUT_CHANNELS = 3
def downsample(filters, size, apply_instancenorm=True):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2D(filters, size, strides=2, padding='same',
kernel_initializer=initializer, use_bias=False))
if apply_instancenorm:
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
result.add(layers.LeakyReLU())
 return result<define_search_model>
plt.style.use('fivethirtyeight')
 training_scores = []
validation_scores = []
test_score_matrix = []
params = study.best_params
params['n_estimators'] = 10000
params['n_jobs'] = -1
for gamma in 0.001*np.arange(0, 100, 1):
params['gamma'] = gamma
model = XGBRegressor(**params)
 train_scores = []
test_scores = []
split = KFold(n_splits=5)
 for train_idx, val_idx in split.split(X_train_fe):
X_tr = X_train_fe.iloc[train_idx]
X_val = X_train_fe.iloc[val_idx]
y_tr = y_train.iloc[train_idx]
y_val = y_train.iloc[val_idx]
model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=30, verbose=0)
 train_score = np.round(np.sqrt(mean_squared_error(y_tr, model.predict(X_tr))), 4)
 test_score = np.round(np.sqrt(mean_squared_error(y_val, model.predict(X_val))), 4)
 train_scores.append(train_score)
 test_scores.append(test_score)
 test_score_matrix.append(test_scores)
 print(f'working on gamma : {gamma}')
 print(f'train scores : {train_scores}')
 print(f'test scores : {test_scores}')
 print(f'Mean OOF RMSLE : {np.mean(test_scores)}')
 train_score = np.mean(train_scores)
 test_score = np.mean(test_scores)
 training_scores.append(train_score)
 validation_scores.append(test_score)
 plt.figure(figsize=(15, 8))
plt.plot(training_scores, 'r-', label='Train-Gamma-Effect')
 plt.plot(validation_scores, 'b-', label='Test-Gamma-Effect')
 plt.legend()
plt.grid()
plt.show()
def upsample(filters, size, apply_dropout=False):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2DTranspose(filters, size, strides=2,
padding='same',
kernel_initializer=initializer,
use_bias=False))
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
if apply_dropout:
result.add(layers.Dropout(0.5))
result.add(layers.ReLU())
 return result<define_search_space>
params = study.best_params
params['n_estimators'] = 1300
params['gamma'] = 0.05
best_params_xgb = params
def Generator() :
inputs = layers.Input(shape=[256,256,3])
 down_stack = [
downsample(64, 4, apply_instancenorm=False),
downsample(128, 4),
downsample(256, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
]
up_stack = [
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4),
upsample(256, 4),
upsample(128, 4),
upsample(64, 4),
]
initializer = tf.random_normal_initializer(0., 0.02)
 last = layers.Conv2DTranspose(OUTPUT_CHANNELS, 4,
strides=2,
padding='same',
kernel_initializer=initializer,
activation='tanh')
 x = inputs
skips = []
for down in down_stack:
x = down(x)
 skips.append(x)
 skips = reversed(skips[:-1])
 for up, skip in zip(up_stack, skips):
x = up(x)
 x = layers.Concatenate()([x, skip])
 x = last(x)
 return keras.Model(inputs=inputs, outputs=x )<define_search_model>
def model_evaluation(model):
cv = RepeatedKFold(n_splits=8, n_repeats=2)
 scores = cross_val_score(model, X_train_fe, y_train, cv=cv, n_jobs=-1, scoring='neg_mean_squared_error')
 return np.sqrt(np.abs(scores))
def Discriminator() :
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 inp = layers.Input(shape=[256, 256, 3], name='input_image')
 x = inp
down1 = downsample(64, 4, False )(x)
 down2 = downsample(128, 4 )(down1)
 down3 = downsample(256, 4 )(down2)
 zero_pad1 = layers.ZeroPadding2D()(down3)
 conv = layers.Conv2D(512, 4, strides=1,
kernel_initializer=initializer,
use_bias=False )(zero_pad1)
 norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init )(conv)
 leaky_relu = layers.LeakyReLU()(norm1)
 zero_pad2 = layers.ZeroPadding2D()(leaky_relu)
 last = layers.Conv2D(1, 4, strides=1,
kernel_initializer=initializer )(zero_pad2)
 return tf.keras.Model(inputs=inp, outputs=last )<define_search_model>
lasso = Lasso(**best_params_lasso)
 ridge = Ridge(**best_params_ridge)
 elastic = ElasticNet(**best_params_elastic)
 svr = SVR(**best_params_svr)
 extra = ExtraTreesRegressor(**best_params_extra)
 random = RandomForestRegressor(**best_params_random)
 gradient = GradientBoostingRegressor(**best_params_gradient)
 xgb = XGBRegressor(**best_params_xgb)
 model_list1 = [('lasso', lasso), 
('svr', svr), 
('extra', extra), 
('elastic', elastic), 
('ridge', ridge), 
('random', random), 
('gradient', gradient), 
('xgb', xgb),
]
with strategy.scope() :
monet_generator = Generator()
photo_generator = Generator()
monet_discriminator = Discriminator()
photo_discriminator = Discriminator()<choose_model_class>
stack_reg = StackingRegressor(estimators=model_list1, cv=8, n_jobs=-1 )
class CycleGan(keras.Model):
def __init__(
 self,
monet_generator,
photo_generator,
monet_discriminator,
photo_discriminator,
lambda_cycle=10, 
):
super(CycleGan, self ).__init__()
self.m_gen = monet_generator
self.p_gen = photo_generator
self.m_disc = monet_discriminator
self.p_disc = photo_discriminator
self.lambda_cycle = lambda_cycle
def compile(
 self,
m_gen_optimizer,
p_gen_optimizer,
m_disc_optimizer,
p_disc_optimizer,
gen_loss_fn,
disc_loss_fn,
cycle_loss_fn,
identity_loss_fn 
):
super(CycleGan, self ).compile()
self.m_gen_optimizer = m_gen_optimizer
self.p_gen_optimizer = p_gen_optimizer
self.m_disc_optimizer = m_disc_optimizer
self.p_disc_optimizer = p_disc_optimizer
self.gen_loss_fn = gen_loss_fn
self.disc_loss_fn = disc_loss_fn
self.cycle_loss_fn = cycle_loss_fn
self.identity_loss_fn = identity_loss_fn
def train_step(self, batch_data):
real_monet, real_photo = batch_data
with tf.GradientTape(persistent=True)as tape:
fake_monet = self.m_gen(real_photo, training=True)
 cycled_photo = self.p_gen(fake_monet, training=True)
 fake_photo = self.p_gen(real_monet, training=True)
 cycled_monet = self.m_gen(fake_photo, training=True)
 same_monet = self.m_gen(real_monet, training=True)
 same_photo = self.p_gen(real_photo, training=True)
 disc_real_monet = self.m_disc(real_monet, training=True)
 disc_real_photo = self.p_disc(real_photo, training=True)
 disc_fake_monet = self.m_disc(fake_monet, training=True)
 disc_fake_photo = self.p_disc(fake_photo, training=True)
 monet_gen_loss = self.gen_loss_fn(disc_fake_monet)
 photo_gen_loss = self.gen_loss_fn(disc_fake_photo)
 total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle)+ self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)
 total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle)
 total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)
 monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)
 photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)
 monet_generator_gradients = tape.gradient(total_monet_gen_loss,
self.m_gen.trainable_variables)
 photo_generator_gradients = tape.gradient(total_photo_gen_loss,
self.p_gen.trainable_variables)
 monet_discriminator_gradients = tape.gradient(monet_disc_loss,
self.m_disc.trainable_variables)
 photo_discriminator_gradients = tape.gradient(photo_disc_loss,
self.p_disc.trainable_variables)
 self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,
self.m_gen.trainable_variables))
self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,
self.p_gen.trainable_variables))
self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,
self.m_disc.trainable_variables))
self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,
self.p_disc.trainable_variables))
return {
"monet_gen_loss": total_monet_gen_loss,
"photo_gen_loss": total_photo_gen_loss,
"monet_disc_loss": monet_disc_loss,
"photo_disc_loss": photo_disc_loss
}<compute_train_metric>
stack_reg.fit(X_train_fe, y_train )
with strategy.scope() :
def discriminator_loss(real, generated):
real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE )(tf.ones_like(real), real)
 generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE )(tf.zeros_like(generated), generated)
 total_disc_loss = real_loss + generated_loss
return total_disc_loss * 0.5<compute_train_metric>
submission = pd.DataFrame(stack_reg.predict(X_test_fe), columns=['SalePrice'] )
with strategy.scope() :
def generator_loss(generated):
return tf.keras.losses.BinaryCrossentropy(from_logits=True,
reduction=tf.keras.losses.Reduction.NONE )(tf.ones_like(generated), generated )<compute_test_metric>
submission = np.expm1(submission )
with strategy.scope() :
def calc_cycle_loss(real_image, cycled_image, LAMBDA):
loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))
return LAMBDA * loss1<compute_test_metric>
submission.insert(loc=0, column='id', value=[i for i in range(1461, 2920)] )
with strategy.scope() :
def identity_loss(real_image, same_image, LAMBDA):
loss = tf.reduce_mean(tf.abs(real_image - same_image))
return LAMBDA * 0.5 * loss<choose_model_class>
submission.to_csv('submission.csv', index=False )
with strategy.scope() :
monet_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 monet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5 )<choose_model_class>
submission.to_csv('submission.csv', index=False )
<train_model><EOS> 
pd.read_csv('submission.csv' )