Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
%matplotlib inline
def warn(*args, **kwargs):
pass
warnings.warn = warn
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_variables>
df_train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 print("There are %0.0f" %df_train.shape[0] + " houses and %0.0f" %df_train.shape[1] +" variables in the training dataset")
 print("There are %0.0f" %df_test.shape[0] + " houses and %0.0f" %df_test.shape[1] +" explanatory variables in the test dataset")
 df_all = df_train.append(df_test)
 df_all.reset_index(inplace=True, drop=True)
 df_all.head()
IMAGE_SIZE = [512, 512]
EPOCHS = 30
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
SEED = 2020
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE<data_type_conversions>
df_all_1 = df_all.copy()
df_all_1 = df_all.drop(['Id','SalePrice'], axis = 1 )
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
def data_augment(image, label):
image = tf.image.random_crop(image, size=[int(IMAGE_SIZE[0]*0.75), int(IMAGE_SIZE[0]*0.75), 3], seed=SEED)
 return image, label
def get_training_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/train/*.tfrec'), labeled=True)
 dataset = dataset.map(data_augment)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(-1)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/val/*.tfrec'), labeled=True, ordered=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(-1)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/test/*.tfrec'), labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(-1)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()<choose_model_class>
df_all_1['MissingCount'] = df_all_1.isnull().sum(axis =1)
 df_all_1['MiscFeature'] = np.where(( df_all.MiscFeature.isnull() == True), 1,0)
 df_all_1['Alley'] = np.where(( df_all.Alley.isnull() == True), 1,0)
 df_all_1['Fence'] = np.where(( df_all.Fence.isnull() == True), 1,0)
 print("There are %0.0f" %df_all_1.shape[1] +" explanatory variables left after removing varibles with more than 50 % missing data" )
LR_START = 0.00001
LR_MAX = 0.0001
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 8
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
remainMiss = missingDesc.drop(['MiscFeature','Alley','Fence','SalePrice'], axis =0)
 df_all_1[remainMiss.index].dtypes
def get_model(use_model):
base_model = use_model(weights='imagenet', include_top=False, pooling='avg', input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(InceptionV3)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 tf.keras.utils.plot_model(
 model, to_file='model.png', show_shapes=True, show_layer_names=True,
)<train_model>
PoolIndex = df_all_1[df_all_1['PoolArea']!=0].index
df_all_1.loc[np.logical_not(df_all_1.index.isin(PoolIndex)) ,'PoolQC'] = 'NA'
df_all_1.loc[PoolIndex,'PoolQC'].isnull().sum()
df_all_1['PoolQC'] = df_all_1['PoolQC'].fillna(df_all_1.PoolQC[PoolIndex].mode() [0])
 df_all_1.PoolQC.value_counts()
history = model.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback, ModelCheckpoint(filepath='my_ef_net_b7.h5', monitor='val_loss',
save_best_only=True)],
validation_data=validation_dataset )<load_pretrained>
FireplaceIndex = df_all_1[df_all_1['Fireplaces']==0].index
df_all_1.loc[df_all_1.index.isin(FireplaceIndex),'FireplaceQu'] = 'NA'
df_all_1.loc[FireplaceIndex,'PoolQC'].isnull().sum()
model = tf.keras.models.load_model('my_ef_net_b7.h5' )<predict_on_test>
cat_length = 30
print('Imputing non-numeric missing variables...')
 df_all_1 = df_all_1.apply(lambda x: x.fillna(x.mode() [0])if(x.dtype == "object" and len(x.unique())<= cat_length)else x, axis=0 )
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )
!pip install -q efficientnet<import_modules>
for col in remainMiss.index:
if(df_all_1[col].dtypes != 'object' and len(np.unique(df_all_1[col])) < cat_length):
print(df_all_1[col].value_counts() )
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2<set_options>
print('Imputing 3 missing numeric variables as categorical varaibles...')
 for col in ['BsmtHalfBath', 'BsmtFullBath','GarageCars']:
df_all_1[col] = df_all_1[col].fillna(df_all_1[col].mode() [0])
 pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )
%matplotlib inline
print("Tensorflow version " + tf.__version__ )<define_variables>
print("Imputing GarageYrBlt...")
 temp = df_all_1[df_all_1.GarageYrBlt.isnull() == True].index
df_all_1["GarageYrBlt"]= np.where(( df_all_1.GarageYrBlt.isnull() == True), df_all_1.YearBuilt,df_all_1.GarageYrBlt ).astype('int64')
 print('Chekcing if GarageYrBlt is corretly imputed...')
(df_all_1.loc[temp,"GarageYrBlt"] == df_all_1.loc[temp,"YearBuilt"] ).unique()
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_variables>
imputeCol = []
for col in df_all_1.columns:
if(df_all_1[col].dtypes != 'object' and len(df_all_1[col].unique())>= cat_length):
imputeCol.append(col)
 print("Columns used for KNN imputation are:
" + str(imputeCol))
imputer = KNNImputer(n_neighbors=5, weights='uniform', metric='nan_euclidean')
 X = df_all_1[imputeCol]
print('Imputing remaining numerical variables with KNN...')
 Xtrans = imputer.fit_transform(X)
 XT = pd.DataFrame(data=Xtrans)
 XT.columns = imputeCol
df_all_1[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']] \
= XT[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']]
print('There are now ' + str(df_all_1.isnull().any().sum())+ ' missing values in our data set.' )
IMAGE_SIZE = [512, 512]
EPOCHS = 30
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
SEED = 2020
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE<normalization>
numVal =(( df_all_1.dtypes != 'object' ).index)[df_all_1.dtypes != 'object']
length = []
for col in numVal:
length.append(len(df_all_1[col].unique()))
numVal_df = pd.DataFrame({'Variables':numVal,'UniqueValuesLength':length})
 num_cat_Val = numVal_df[numVal_df['UniqueValuesLength'] < cat_length].sort_values(by='UniqueValuesLength', ascending=False)
 num_cat_Val.head()
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
def data_augment(image, label):
image = tf.image.random_crop(image, [150, 150, 3],seed=SEED)
 return image, label
def get_training_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/train/*.tfrec'), labeled=True)
 dataset = dataset.map(data_augment)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(-1)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/val/*.tfrec'), labeled=True, ordered=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(-1)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/test/*.tfrec'), labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(-1)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()<choose_model_class>
print('There are %i' %numVal_df[numVal_df['UniqueValuesLength'] < cat_length].shape[0] + ' numerical variables and %i' \
%(df_all_1.dtypes == 'object' ).sum() + ' non-numerical variables can potentially be treated as categorcial variables with some exceptions.' )
LR_START = 0.00001
LR_MAX = 0.0001
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 9
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
cont_val = numVal_df[numVal_df['UniqueValuesLength'] >= cat_length].sort_values(by='UniqueValuesLength')
 cont_val.head()
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(InceptionV3)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 tf.keras.utils.plot_model(
 model, to_file='model.png', show_shapes=True, show_layer_names=True,
)<train_model>
yCor = cor['SalePrice'].sort_values(ascending = False ).reset_index().drop(index=0)
 yCor.columns = ["Variables", "Cor"]
yCor[(yCor["Cor"] > 0.5)|(yCor["Cor"] < -0.5)]
history = model.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback, ModelCheckpoint(filepath='my_ef_net_b7.h5', monitor='val_loss',
save_best_only=True)],
validation_data=validation_dataset )<load_pretrained>
df_all_red = df_all_1.copy()
model = tf.keras.models.load_model('my_ef_net_b7.h5' )<predict_on_test>
df_all_red.YearRemodAdd = np.where(df_all_1.YearBuilt >df_all_1.YearRemodAdd, df_all_1.YearBuilt, df_all_1.YearRemodAdd)
 df_all_red.YrSold = np.where(df_all_1.YearBuilt >df_all_1.YrSold , df_all_1.YearBuilt, df_all_1.YrSold)
 df_all_red.YrSold = np.where(df_all_red.YearRemodAdd > df_all_red.YrSold, df_all_1.YearRemodAdd, df_all_red.YrSold)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt < df_all_red.YearBuilt, df_all_red.YearBuilt, df_all_red.GarageYrBlt)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt >2010, df_all_red.YearBuilt,df_all_red.GarageYrBlt)
 df_all_red['RestorePeriod'] = df_all_red.YearRemodAdd - df_all_red.YearBuilt
df_all_red['HouseAge'] = df_all_red.YrSold - df_all_red.YearBuilt
df_all_red["RemodAge"] = df_all_red.YrSold - df_all_red.YearRemodAdd
df_all_red['GarageAge'] = df_all_red.YrSold - df_all_red.GarageYrBlt
df_all_red["IsNewHouse"] = np.where(df_all_red.YearBuilt == df_all_red.YrSold, 1 ,0)
 df_all_red["Remodeled"] = np.where(df_all_red.RestorePeriod != 0, 1 ,0)
 df_all_red["GreatDepression"] = np.where(( df_all_1.YearBuilt >=1929)&(df_all_1.YearBuilt <= 1939), 1,0)
 df_all_red["Pre-GreatDepression"] = np.where(( df_all_1.YearBuilt <1929), 1,0)
 df_all_red["MortgageCrisis"] = np.where(( df_all_1.YearBuilt >=2007)&(df_all_1.YearBuilt <= 2010), 1,0)
 df_all_red["MortgageCrisisRestoration"] = np.where(( df_all_1.YearRemodAdd >=2007)&(df_all_1.YearRemodAdd <= 2010), 1,0)
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
df_all_red.drop(["YearBuilt","GarageYrBlt", "YearRemodAdd"], axis = 1, inplace = True )
!pip install -q efficientnet<import_modules>
df_all_red["TotalFlrSF"] = df_all_red["1stFlrSF"] + df_all_red["2ndFlrSF"]
df_all_red["TotalBsmtFin"] = df_all_red['BsmtFinSF1']+df_all_red['BsmtFinSF2']
df_all_red["BsmtFinRatio"] =(df_all_red['TotalBsmtFin'] / df_all_red['TotalBsmtSF'] ).fillna(0)
 df_all_red["IsBasement"] = np.where(df_all_red['TotalBsmtSF'] < 1 , 0, 1)
 df_all_red["PorchArea"] = df_all_red['OpenPorchSF'] + df_all_red['EnclosedPorch'] + df_all_red['ScreenPorch'] + \
df_all_red["3SsnPorch"] + df_all_red['WoodDeckSF']
df_all_red["TotalHouseArea"] = df_all_red['TotalFlrSF'] + df_all_red['TotalBsmtSF']
df_all_red["TotalLivArea"] = df_all_red['GrLivArea'] + df_all_red['TotalBsmtSF']
df_all_red["2ndFloor"] = np.where(( df_all_red['2ndFlrSF'] < 1), 0, 1)
 df_all_red["GarageAreaPerCar"] =(df_all_red.GarageArea / df_all_red.GarageCars ).fillna(0)
 df_all_red["HouseLotRatio"] = df_all_red.TotalHouseArea /df_all_red.LotArea
df_all_red["LivLotRatio"] = df_all_red.GrLivArea / df_all_red.LotArea
df_all_red["GarageLotRatio"] = df_all_red.GarageArea / df_all_red.LotArea
df_all_red["MasVnrRatio"] = df_all_red.MasVnrArea / df_all_red.TotalHouseArea
df_all_red["DifArea"] =(df_all_red.LotArea - df_all_red["1stFlrSF"] - df_all_red.GarageArea - df_all_red.PorchArea)
 df_all_red["LowQualFinSFRatio"] = df_all_red.LowQualFinSF/df_all_red.TotalHouseArea
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2, EfficientNetB0, EfficientNetB1<set_options>
for col in ['TotRmsAbvGrd', 'OverallQual', 'OverallCond','BedroomAbvGr','GarageCars','KitchenAbvGr','FullBath','MissingCount']:
print(col + ' has ' + str(len(df_all_red[col].value_counts())) + ' unique numerical values.' )
%matplotlib inline
print("Tensorflow version " + tf.__version__ )<define_variables>
for col in ['OverallQual','OverallCond']:
df_all_red[col].replace(np.linspace(1,10,10 ).astype(int), list(np.linspace(10,100,10)) , inplace=True)
 df_all_red['OverallScore'] = df_all_red[['OverallQual', 'OverallCond']].sum(axis = 1)/2
df_all_red["OverallGrade"] = df_all_red["OverallQual"] * df_all_red["OverallCond"]
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_variables>
for col in ['BsmtFinType1','BsmtFinType2']:
df_all_red[col].replace(to_replace=['Unf', 'LwQ','Rec','BLQ','ALQ','GLQ'], value=[1/6,2/6,3/6,4/6,5/6,1], inplace=True)
 for col in ['BsmtCond', 'BsmtQual', 'GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual','HeatingQC','FireplaceQu','PoolQC']:
df_all_red[col].replace(to_replace=['NA','Po','Fa', 'TA','Gd','Ex'], value=[0,1/5,2/5,3/5,4/5,1], inplace=True)
 df_all_red["Functional"].replace(['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],[1/7,2/7,3/7,4/7,5/7,6/7,1] , inplace = True)
 df_all_red["BsmtExposure"].replace(['No','Mn','Av','Gd'],[1/4,2/4,3/4,1] , inplace = True)
 ratingCols= ['BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','BsmtExposure','GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual', \
"HeatingQC","Functional",'FireplaceQu','PoolQC']
df_all_red[ratingCols] = df_all_red[ratingCols]*100
df_all_red['BsmtScore']= df_all_red[["BsmtCond", "BsmtQual","BsmtExposure","BsmtFinType1", "BsmtFinType2"]].sum(axis = 1)/5
df_all_red['GarageScore']=df_all_red[["GarageQual", "GarageCond"]].sum(axis = 1)/2
df_all_red['ExterScore'] = df_all_red[["ExterQual", "ExterCond"]].sum(axis = 1)/2
df_all_red['QualScore'] = df_all_red[["OverallQual", "ExterQual", "GarageQual", "BsmtQual","BsmtFinType1", "BsmtFinType2", "KitchenQual","PoolQC", \
"FireplaceQu"]].sum(axis = 1)/9
df_all_red['CondScore'] = df_all_red[["OverallCond", "ExterCond", "GarageCond", "BsmtCond", "HeatingQC", "Functional"]].sum(axis = 1)/6
df_all_red['GarageGrade'] = df_all_red["GarageQual"] * df_all_red["GarageCond"]
df_all_red['ExterGrade'] = df_all_red["ExterQual"] * df_all_red["ExterCond"]
df_all_red["KitchenGrade"] = df_all_red["KitchenAbvGr"] * df_all_red["KitchenQual"]
df_all_red['BsmtGrade'] = df_all_red["BsmtCond"] * df_all_red["BsmtQual"]
df_all_red['FireplaceGrade'] = df_all_red["Fireplaces"] * df_all_red["FireplaceQu"]
df_all_red['PoolGrade'] = df_all_red["PoolArea"] * df_all_red["PoolQC"]
IMAGE_SIZE = [512, 512]
EPOCHS = 30
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SEED = 2020<data_type_conversions>
for col in num_cat_Val['Variables']:
print(df_all_red[col].value_counts() )
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
flag = random.randint(1,3)
 coef_1 = random.randint(70, 90)* 0.01
coef_2 = random.randint(70, 90)* 0.01
if flag == 1:
image = tf.image.random_flip_left_right(image, seed=SEED)
 elif flag == 2:
image = tf.image.random_flip_up_down(image, seed=SEED)
 else:
image = tf.image.random_crop(image, [int(IMAGE_SIZE[0]*coef_1), int(IMAGE_SIZE[0]*coef_2), 3],seed=SEED)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<init_hyperparams>
df_all_red["MSSubClass_1-Story"] = np.where(df_all_red.MSSubClass.isin([20,30,40,120]), 1, 0)
 df_all_red["MSSubClass_1-1/2Story"] = np.where(df_all_red.MSSubClass.isin([45,50,150]), 1, 0)
 df_all_red["MSSubClass_2-Story"] =np.where(df_all_red.MSSubClass.isin([60,70,160]), 1, 0)
 df_all_red["MSSubClass_2-1/2Story"] = np.where(df_all_red.MSSubClass.isin([75]), 1, 0)
 df_all_red["MSSubClass_Split"] = np.where(df_all_red.MSSubClass.isin([80,85]), 1, 0)
 df_all_red["MSSubClass_AllAges"] = np.where(df_all_red.MSSubClass.isin([40,45,50,75,90,150, 190]), 1, 0)
 df_all_red["MSSubClass_Newer"] = np.where(df_all_red.MSSubClass.isin([20, 60, 120, 160]), 1, 0)
 df_all_red["MSSubClass_Older"] = np.where(df_all_red.MSSubClass.isin([30, 70]), 1, 0)
 df_all_red["MSSubClass_AllStyles"] = np.where(df_all_red.MSSubClass.isin([20, 90, 190]), 1, 0)
 df_all_red["MSSubClass_Pud"] = np.where(df_all_red.MSSubClass.isin([150, 160, 180]), 1, 0)
 df_all_red["MSSubClass_SplitMultiLevel"] = np.where(df_all_red.MSSubClass.isin([80,85, 180]), 1, 0)
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.75
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
df_all_red.drop(["MSSubClass"], axis = 1, inplace = True )
def get_model(use_model):
base_model = use_model(weights='noisy-student',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(EfficientNetB7)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
<train_model>
print("Collapsing categories...")
 df_all_red['LotShape'] = np.where(df_all_1.LotShape != 'Reg', 'IR', df_all_1['LotShape'])
 df_all_red['LandContour'] = np.where(df_all_1.LandContour != 'Lvl', 'IR', df_all_1['LandContour'])
 df_all_red['RoofMatl'] = np.where(df_all_1.RoofMatl != 'CompShg', 'Non-Standard', df_all_1['RoofMatl'])
 df_all_red['Heating'] = np.where(df_all_1.Heating.str.contains("Gas"), 1, 0)
 df_all_red["MSZoning"] = np.where(df_all_red.MSZoning.isin(["RH", "RM", "RL"]), 1, 0)
 df_all_red['Electrical'] = np.where(df_all_1.Electrical != 'SBrkr', 'non-Standard', df_all_1['Electrical'])
 df_all_red['SaleCondition'] = np.where(df_all_1.SaleCondition != 'Normal', 'non-Normal', df_all_1['SaleCondition'])
 df_all_red["SaleType"] = np.where(df_all_1.SaleType.isin(["Con", "ConLD", "ConLI", "ConLw"]), "Contract", df_all_1.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["WD", "CWD"]), "Warranty", df_all_red.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["COD"]), "Oth", df_all_red.SaleType)
 df_all_red["Foundation"] = np.where(df_all_1["Foundation"] == "Stone", "BrkTil", df_all_1["Foundation"])
 df_all_red["Foundation"] = np.where(df_all_red["Foundation"] == "Wood", "PConc", df_all_red["Foundation"])
 df_all_red["BldgType"] = np.where(df_all_1.BldgType.isin(['1Fam', '2fmCon']), "Family", df_all_1.BldgType)
 df_all_red["BldgType"] = np.where(df_all_red.BldgType.isin(['TwnhsE', 'Twnhs']), "Townhouse", df_all_red.BldgType )
history = model.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback, ModelCheckpoint(filepath='my_ef_net_b7.h5', monitor='val_loss',
save_best_only=True)],
validation_data=get_validation_dataset() ,
workers = 3 )<load_pretrained>
df_all_red["CNormal"] = np.where(( df_all_1.Condition1.isin(["Norm"])) |(df_all_1.Condition2.isin(["Norm"])) , 1, 0)
 df_all_red["CRailroad"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) , 1, 0)
 df_all_red["CPark"] = np.where(( df_all_1.Condition1.isin(["PosN", "PosA"])) |(df_all_1.Condition2.isin(["PosN", "PosA"])) , 1, 0)
 df_all_red["CAdjacent"] = np.where(( df_all_1.Condition1.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) |(df_all_1.Condition2.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) , 1, 0)
 df_all_red["CWithin"] = np.where(( df_all_1.Condition1.isin(["RRNn", "PosN", "RRNe"])) |(df_all_1.Condition2.isin(["RRNn", "PosN", "RRNe"])) , 1, 0)
 df_all_red["CNorthS"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn"])) , 1, 0)
 df_all_red["CEastW"] = np.where(( df_all_1.Condition1.isin(["RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNe", "RRAe"])) , 1, 0)
 df_all_red["CStreet"] = np.where(( df_all_1.Condition1.isin(["Artery", "Feedr"])) |(df_all_1.Condition2.isin(["Artery", "Feedr"])) , 1, 0)
 df_all_red["HouseStyle_Floor"] = np.where(df_all_1.HouseStyle.isin(["1Story"]), "1", "2")
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["1.5Fin", "1.5Unf"]), "1.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["2.5Fin", "2.5Unf"]), "2.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["SLvl", "SFoyer"]), "Split", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_IsSplit"] = np.where(df_all_red.HouseStyle_Floor == 'Split',1,0)
 df_all_red['HasMasVnrType'] = np.where(df_all_1['MasVnrType'] == "None",0,1 )
model = tf.keras.models.load_model('my_ef_net_b7.h5' )<predict_on_test>
df_all_red.drop(["Condition1", "Condition2", "YrSold"], axis = 1, inplace = True )
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<set_options>
df_all_2 = df_all_red.copy()
print(tf.__version__)
 warnings.filterwarnings('ignore' )<set_options>
labelencoder = preprocessing.LabelEncoder()
label_cols = [col for col in df_all_red.columns if len(df_all_red[col].value_counts())== 2]
for col in label_cols:
df_all_2[col] = labelencoder.fit_transform(df_all_red[col] )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Device:', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except:
strategy = tf.distribute.get_strategy()
print("No tpu present")
 print('Number of replicas:', strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE<define_variables>
cat_cols = df_all_2.columns[(df_all_2.dtypes== 'object')|(df_all_2.dtypes =='category')]
print("There are " + str(len(cat_cols)) +" categorical variables require one hot encoding.")
 cat_cols
IMAGE_SIZE = [224, 224]
EPOCHS = 10
FOLDS = 5
SEED = 777
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<define_variables>
print("One hot encoding...")
 original_columns = list(df_all_2.columns)
 df_final= pd.get_dummies(df_all_2, columns=cat_cols, dummy_na=False, drop_first=True)
 new_columns = [c for c in df_final.columns if c not in original_columns]
print("There are " + str(len(new_columns)) + " variables have been created after one hot encoding" )
GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')
 GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')+ tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )<define_variables>
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<choose_model_class>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(50 if EPOCHS<50 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<categorify>
df_final['SalePrice'] = df_all['SalePrice']
df_final['ID'] = df_all['Id']
print("The final dataset has " + str(df_final.shape[0])+ " observations with " + str(df_final.shape[1]-2)+ " explanatory variables and 1 response variable")
 df_final.head(5 )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset,do_aug=True):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<normalization>
train = df_final[df_final.ID.isin(df_train.Id)]
test = df_final[df_final.ID.isin(df_test.Id)]
X = train.drop(['SalePrice', "ID"], axis=1)
 y = np.array(train['SalePrice'])
 k_fold = KFold(n_splits=5, shuffle=True, random_state=79 )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))<categorify>
xgb_params = {'alpha': 13, 'colsample_bytree': 0.4,
'gamma': 0, 'lambda': 1,
'learning_rate': 0.1,
'max_depth': 4,
'n_estimators': 700,
'subsample': 0.8}
lgbm_params = {'subsample': 0.8, 'num_leaves': 50,
'n_estimators': 4000,
'min_gain_to_split': 10,
'max_depth': 5, 'max_bin': 40,
'learning_rate': 0.01,
'lambda_l1': 0.0 ,'lambda_l2': 0.6000000000000001,
'feature_fraction': 0.16000000000000003
}
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label 
<choose_model_class>
i = 0
train_pred_XGB = np.zeros(X.shape[0])
 train_pred_LGBM = np.zeros(X.shape[0])
 test_pred_XGB = []
test_pred_LGBM = []
fold = ['1st', '2nd', '3rd', '4th', '5th']
for train_index, test_index in k_fold.split(X):
print("TRAIN:", train_index[:10], "TEST:", test_index[:10])
 X_train, X_test = X.loc[train_index,:], X.loc[test_index,:]
y_train, y_test = y[train_index], y[test_index]
print("Train and make prediction for the {} Fold...".format(fold[i]))
xgb_model = xgb.XGBRegressor(random_state=79,booster='gbtree',
objective ="reg:squarederror",
**xgb_params)
 xgb_model.fit(X_train,y_train)
 lgbm_model = lgb.LGBMRegressor(random_state=79, objective = "regression",
metric = "rmse",**lgbm_params)
 lgbm_model.fit(X_train,y_train)
 train_pred_XGB[test_index] = xgb_model.predict(X_test)
 train_pred_LGBM[test_index] = lgbm_model.predict(X_test)
 test_pred_XGB.append(xgb_model.predict(test.drop(["ID","SalePrice"],
axis = 1)))
 test_pred_LGBM.append(lgbm_model.predict(test.drop(["ID","SalePrice"],
axis = 1)))
 i += 1
def get_model() :
with strategy.scope() :
rnet = InceptionResNetV2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def train_cross_validate(folds = 5):
histories = []
models = []
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('
print('
print('
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback],
validation_data = get_validation_dataset(val_dataset),
verbose=2 
)
 models.append(model)
 histories.append(history)
 return histories, models
def train_and_predict(folds = 5):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Start training %i folds'%folds)
 histories, models = train_cross_validate(folds = folds)
 print('Computing predictions...')
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(folds)], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 return histories, models
histories_inception, inception_model = train_and_predict(folds = FOLDS )<choose_model_class>
Stacking_Train = pd.DataFrame({'XGBoost': train_pred_XGB, 'LightGBM':train_pred_LGBM,
'SalePrice': train.SalePrice})
 X = Stacking_Train.iloc[:,:2]
y = np.array(Stacking_Train['SalePrice'])
 Stacking_Test = pd.DataFrame({'XGBoost': sum(test_pred_XGB)/5,
'LightGBM':sum(test_pred_LGBM)/5,
} )
def get_model() :
with strategy.scope() :
rnet = NASNetLarge(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def train_cross_validate(folds = 5):
histories = []
models = []
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('
print('
print('
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback],
validation_data = get_validation_dataset(val_dataset),
verbose=2 
)
 models.append(model)
 histories.append(history)
 return histories, models
def train_and_predict(folds = 5):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Start training %i folds'%folds)
 histories, models = train_cross_validate(folds = folds)
 print('Computing predictions...')
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(folds)], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 return histories, models 
<split>
k_fold = RepeatedKFold(n_splits=5, n_repeats = 3, random_state=79)
 Final_est = ElasticNet(random_state = 79)
 Final_params = {'alpha': [100,1000],
'l1_ratio': np.arange(0, 1, 0.01)
 }
Final_model = GridSearchCV(Final_est, Final_params, scoring='neg_mean_absolute_error',
cv=k_fold, n_jobs=-1, verbose = 1)
 Final_model.fit(X,y)
 print(Final_model.best_params_)
 print("The CV Error is: " + str(-Final_model.best_score_))
best_model = Final_model.best_estimator_
%%time
all_labels = []; all_prob = []; all_pred = []
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for j,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('Inferring fold',j+1,'validation images...')
 VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES'])
 NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)
 cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True))
images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
all_labels.append(next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy())
 prob = inception_model[j].predict(images_ds)
 all_prob.append(prob)
 all_pred.append(np.argmax(prob, axis=-1))
cm_correct_labels = np.concatenate(all_labels)
 cm_probabilities = np.concatenate(all_prob)
 cm_predictions = np.concatenate(all_pred )<compute_test_metric>
k_fold = KFold(n_splits=3, shuffle=True, random_state=79)
 i = 0
train_pred_best = np.zeros(X.shape[0])
 test_pred_best = []
rmseStack = np.zeros(3)
 fold = ['1st', '2nd', '3rd']
for train_index, test_index in k_fold.split(X):
print("TRAIN:", train_index[:10], "TEST:", test_index[:10])
 X_train, X_test = X.loc[train_index,:], X.loc[test_index,:]
y_train, y_test = y[train_index], y[test_index]
print("Train and make prediction for the {} Fold...".format(fold[i]))
best_model.fit(X_train,y_train)
 train_pred_best[test_index] = best_model.predict(X_test)
 rmseStack[i] = round(np.sqrt(mean_squared_error(y, best_model.predict(X))),2)
 test_pred_best.append(best_model.predict(Stacking_Test))
i += 1
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall)) ; print()<create_dataframe>
cv_score = np.mean(cross_val_score(best_model,X,y,scoring = 'neg_root_mean_squared_error', cv=k_fold,n_jobs = -1,verbose =0))
print('XGBoost RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred_XGB)) ,2))
print('LGBM RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred_LGBM)) ,2))
print('Model Stacking RMSE for training set :',round(np.sqrt(mean_squared_error(y, train_pred_best)) ,2))
print('Model Stacking Cross Validation RMSE for training set :', round(-cv_score,2))
<set_options>
y_pred_final = sum(test_pred_best)/3
print("TF version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE<set_options>
submission = pd.DataFrame({"Id": test.ID, "SalePrice": y_pred_final})
 submission.to_csv("submission.csv", index = None)
 submission