Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
val_probabilities = [0] * no_of_models
test_probabilities = [0] * no_of_models
all_probabilities = [0] * no_of_models<choose_model_class>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.shape
with strategy.scope() :
for j in range(no_of_models):
models[j] = create_EfficientNet_model()
models[0].summary()<load_pretrained>
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test.shape
def write_history(j):
history_dict = [0] * no_of_models
for i in range(j + 1):
if(historys[i] != 0):
history_dict[i] = historys[i].history
filename = './' + this_run_file_prefix + 'model_history_' + str(j)+ '.pkl'
pklfile = open(filename, 'ab')
 pickle.dump(history_dict, pklfile)
 pklfile.close()<define_variables>
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission.shape
EPOCHS = 20
historys = [0] * no_of_models
finished_models = 0
for j in range(start_model, end_model):
start_training = datetime.now()
print(start_training)
 time_from_start_program_tdelta = start_training - start_time
if time_from_start_program_tdelta > end_training_by_tdelta:
print(j, 'time limit for doing training over, get out')
 break
print('LR_EXP_DECAY:', LR_EXP_DECAY, '.LR_MAX:', LR_MAX)
 historys[j] = models[j].fit(get_training_dataset() , steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS, validation_data = get_validation_dataset() , callbacks = [lr_callback, early_stop])
 write_history(j)
 filename = this_run_file_prefix + 'models_' + str(j)+ '.h5'
models[j].save(filename)
 gc.collect()
finished_models = j + 1
print(datetime.now())
<categorify>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
cmdataset = get_validation_dataset(ordered = True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()<create_dataframe>
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True )
test_ds = get_test_dataset(ordered = True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
<define_variables>
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
dataset = get_validation_dataset()
dataset = dataset.unbatch().batch(20)
 batch = iter(dataset)
 images, labels = next(batch )<predict_on_test>
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 print(features.shape)
 features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str )
print(datetime.now())
 for j in range(start_model, end_model):
val_probabilities[j] = models[j].predict(images_ds)
 test_probabilities[j] = models[j].predict(test_images_ds)
 all_probabilities[j] = models[j].predict(images)
 print(datetime.now())
<find_best_params>
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None" )
for j in range(start_model, finished_models):
display_training_curves(historys[j].history['loss'], historys[j].history['val_loss'], 'loss', 211)
 display_training_curves(historys[j].history['sparse_categorical_accuracy'], historys[j].history['val_sparse_categorical_accuracy'], 'accuracy', 212)
 for j in range(start_model, finished_models):
print('model number:', j, ', Train Accuracy:', max(historys[j].history['sparse_categorical_accuracy']), ', Validation Accuracy:', max(historys[j].history['val_sparse_categorical_accuracy']))
for j in range(start_model, finished_models):
print('model number:', j, ', Train Loss:', min(historys[j].history['loss']), ', Validation Loss:', min(historys[j].history['val_loss']))<compute_test_metric>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None' )
def getFitPrecisionRecall(correct_labels, predictions):
score = f1_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 precision = precision_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 recall = recall_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 return score, precision, recall<compute_test_metric>
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)))
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, cm_predictions)
 cmat =(cmat.T / cmat.sum(axis = -1)).T
display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<define_variables>
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i )
def create_submission_file(filename, probabilities):
predictions = np.argmax(probabilities, axis = -1)
 print('Generating submission file...', filename)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt(filename, np.rec.fromarrays([test_ids, predictions]), fmt = ['%s', '%d'], delimiter = ',', header = 'id,label', comments = '')
<save_to_csv>
features.update(features[objects].fillna('None'))
probabilities = np.zeros(( test_probabilities[0].shape))
for j in range(no_of_models):
probabilities = probabilities + test_probabilities[j]
filename = this_run_file_prefix + 'submission.csv'
create_submission_file(filename, probabilities)
 create_submission_file('submission.csv', probabilities )<find_best_params>
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
def combine_two(correct_labels, probability_0, probability_1):
print('Start.', datetime.now())
 alphas0_to_try = np.linspace(0, 1, 101)
 best_score = -1
best_alpha0 = -1
best_alpha1 = -1
best_precision = -1
best_recall = -1
best_val_predictions = None
for alpha0 in alphas0_to_try:
alpha1 = 1.0 - alpha0
probabilities = alpha0 * probability_0 + alpha1 * probability_1
predictions = np.argmax(probabilities, axis = -1)
 score, precision, recall = getFitPrecisionRecall(correct_labels, predictions)
 if score > best_score:
best_alpha0 = alpha0
best_alpha1 = alpha1
best_score = score
best_precision = precision
best_recall = recall
best_val_predictions = predictions
return best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall<define_search_space>
skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False )
def combine_three(correct_labels, probability_0, probability_1, probability_2):
print('Start.', datetime.now())
 alphas0_to_try = np.linspace(0, 1, 101)
 alphas1_to_try = np.linspace(0, 1, 101)
 best_score = -1
best_alpha0 = -1
best_alpha1 = -1
best_alpha2 = -1
best_precision = -1
best_recall = -1
best_val_predictions = None
for alpha0 in alphas0_to_try:
for alpha1 in alphas1_to_try:
if(alpha0 + alpha1)> 1.0:
break
alpha2 = 1.0 - alpha0 - alpha1
probabilities = alpha0 * probability_0 + alpha1 * probability_1 + alpha2 * probability_2
predictions = np.argmax(probabilities, axis = -1)
 score, precision, recall = getFitPrecisionRecall(correct_labels, predictions)
 if score > best_score:
best_alpha0 = alpha0
best_alpha1 = alpha1
best_alpha2 = alpha2
best_score = score
best_precision = precision
best_recall = recall
best_val_predictions = predictions
return best_alpha0, best_alpha1, best_alpha2, best_val_predictions, best_score, best_precision, best_recall<train_model>
high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
def get_best_combination(no_models, cm_correct_labels, val_probabilities, test_probabilities):
best_fit_score = -10000.0
best_predictions = 0
choose_filename = ''
curr_predictions = np.argmax(val_probabilities[0], axis = -1)
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, curr_predictions)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
filename = this_run_file_prefix + 'submission_0.csv'
if best_fit_score < score:
best_fit_score = score
best_predictions = curr_predictions
choose_filename = filename
create_submission_file('./submission.csv', test_probabilities[0])
 create_submission_file(filename, test_probabilities[0])
 if no_models > 1:
curr_predictions = np.argmax(val_probabilities[1], axis = -1)
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, curr_predictions)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
filename = this_run_file_prefix + 'submission_1.csv'
if best_fit_score < score:
best_fit_score = score
best_predictions = curr_predictions
choose_filename = filename
create_submission_file('./submission.csv', test_probabilities[1])
 create_submission_file(filename, test_probabilities[1])
 if no_models > 2:
curr_predictions = np.argmax(val_probabilities[2], axis = -1)
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, curr_predictions)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
filename = this_run_file_prefix + 'submission_2.csv'
if best_fit_score < score:
best_fit_score = score
best_predictions = curr_predictions
choose_filename = filename
create_submission_file('./submission.csv', test_probabilities[2])
 create_submission_file(filename, test_probabilities[2])
 if no_models > 1:
best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall = combine_two(cm_correct_labels, val_probabilities[0], val_probabilities[1])
 print('For indx', [0, 1], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[0] + best_alpha1 * test_probabilities[1]
filename = this_run_file_prefix + 'submission_01.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 if no_models > 2:
best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall = combine_two(cm_correct_labels, val_probabilities[0], val_probabilities[2])
 print('For indx', [0, 2], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[0] + best_alpha1 * test_probabilities[2]
filename = this_run_file_prefix + 'submission_02.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall = combine_two(cm_correct_labels, val_probabilities[1], val_probabilities[2])
 print('For indx', [1, 2], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[1] + best_alpha1 * test_probabilities[2]
filename = this_run_file_prefix + 'submission_12.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 best_alpha0, best_alpha1, best_alpha2, best_val_predictions, best_score, best_precision, best_recall = combine_three(cm_correct_labels, val_probabilities[0], val_probabilities[1], val_probabilities[2])
 print('For indx', [0, 1, 2], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, 'best_alpha2:', best_alpha2, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[0] + best_alpha1 * test_probabilities[1] + best_alpha2 * test_probabilities[2]
filename = this_run_file_prefix + 'submission_012.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 cmat = confusion_matrix(cm_correct_labels, best_predictions, labels = range(len(CLASSES)))
 cmat =(cmat.T / cmat.sum(axis = -1)).T
display_confusion_matrix(cmat, best_fit_score, precision, recall)
 print('Best score from all combination was', best_fit_score, '.For submission file used is', choose_filename)
 return best_predictions 
<categorify>
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
best_predictions = cm_predictions
if no_of_models > 1:
bp = get_best_combination(no_of_models, cm_correct_labels, val_probabilities, test_probabilities)
 best_predictions = bp 
<define_variables>
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1 )
probabilities = np.zeros(( all_probabilities[0].shape))
for j in range(no_of_models):
probabilities = probabilities + all_probabilities[j]
predictions = np.argmax(probabilities, axis =-1)
 display_batch_of_images(( images, labels), predictions )<load_pretrained>
features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
val_probs = [cm_correct_labels, cm_predictions, val_probabilities[0], val_probabilities[1], test_probabilities[0], test_probabilities[1]]
filename = this_run_file_prefix + 'tests_vals_01.pkl'
pklfile = open(filename, 'ab')
 pickle.dump(val_probs, pklfile)
 pklfile.close()<categorify>
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'] )
use_correct_labels = cm_correct_labels
use_val_predictions = best_predictions<define_variables>
features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
correct_labels_cnt = 0
incorrect_labels_cnt = 0
correct_labels = []
incorrect_labels = []
vals_actual_true = {}
vals_tp = {}
vals_fn = {}
vals_fp = {}
for i in range(len(CLASSES)) :
vals_actual_true[i] = 0
vals_tp[i] = 0
vals_fn[i] = 0
vals_fp[i] = 0
for i in range(len(use_correct_labels)) :
correct_label = use_correct_labels[i]
predict_label = use_val_predictions[i]
vals_actual_true[correct_label] = vals_actual_true[correct_label] + 1
if use_val_predictions[i] != use_correct_labels[i]:
incorrect_labels_cnt = incorrect_labels_cnt + 1
incorrect_labels.append(i)
 vals_fn[correct_label] = vals_fn[correct_label] + 1
vals_fp[predict_label] = vals_fp[predict_label] + 1
else:
correct_labels_cnt = correct_labels_cnt + 1
correct_labels.append(i)
 vals_tp[correct_label] = vals_tp[correct_label] + 1
print('Number of correct_labels is {}, incorrect_labels is {}'.format(correct_labels_cnt, incorrect_labels_cnt))
print('Incorrect labels', incorrect_labels)
<define_variables>
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'] )
def display_my_batch_of_images(databatch, rows = 0, cols = 0, predictions=None):

 images, labels = databatch
if labels is None:
labels = [None for _ in enumerate(images)]
if rows == 0 or cols == 0:
rows = int(math.sqrt(len(images)))
 cols =(len(images)+ rows - 1)//rows
print('Total number of images is {}, rows is {}, cols is {}'.format(len(images), rows, cols))
FIGSIZE = 20.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()<define_variables>
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
disp_labels = []
disp_predictions = []
for i in range(54):
if i >= incorrect_labels_cnt:
break
id = incorrect_labels[i]
disp_labels.append(use_correct_labels[id])
 disp_predictions.append(use_val_predictions[id])
 print(disp_labels)
 print(disp_predictions )<load_from_csv>
print(features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape )
val_ids = list(range(len(use_correct_labels)))
 filename = this_run_file_prefix + 'validation_results.csv'
np.savetxt(filename, np.rec.fromarrays([val_ids, use_correct_labels, use_val_predictions]), fmt = ['%d', '%d', '%d'], delimiter = ',', header = 'id,correct_label,predicted_label', comments = '')
 cls_ids = list(range(len(CLASSES)))
 filename = this_run_file_prefix + 'validation_statistics.csv'
np.savetxt(filename, np.rec.fromarrays([cls_ids, list(vals_actual_true.values()), list(vals_tp.values()), list(vals_fn.values()), list(vals_fp.values())]), fmt = ['%d', '%d', '%d', '%d', '%d'], delimiter = ',', header = 'cls_id,actual_true,true_positive,false_negative,false_positive', comments = '' )<set_options>
X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(X):, :]
print("TF version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE<set_options>
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i )
IMAGE_SIZE = [512, 512]
EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
SEED = 752
SKIP_VALIDATION = False
TTA_NUM = 5
random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )<categorify>
overfit = list(overfit)
 overfit.append('MSZoning_C(all)' )
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )<categorify>
X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()
def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):
p=random.random()
if p>=0.25:
w, h, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3
origin_area = tf.cast(h*w, tf.float32)
 e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)) , tf.int32)
 e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)) , tf.int32)
 e_height_h = tf.minimum(e_size_h, h)
 e_width_h = tf.minimum(e_size_h, w)
 erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)
 erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)
 erase_area = tf.zeros(shape=[erase_height, erase_width, c])
 erase_area = tf.cast(erase_area, tf.uint8)
 pad_h = h - erase_height
pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)
 pad_bottom = pad_h - pad_top
pad_w = w - erase_width
pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)
 pad_right = pad_w - pad_left
erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)
 erase_mask = tf.squeeze(erase_mask, axis=0)
 erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))
return tf.cast(erased_img, img.dtype)
 else:
return tf.cast(img, img.dtype )<categorify>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42 )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image, seed=SEED)
 image = random_blockout(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<define_variables>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<train_model>
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse )
print("Training data shapes:")
 for image, label in get_training_dataset().take(3):
print(image.numpy().shape, label.numpy().shape)
 print("Training data label examples:", label.numpy())
 print("Validation data shapes:")
 for image, label in get_validation_dataset().take(3):
print(image.numpy().shape, label.numpy().shape)
 print("Validation data label examples:", label.numpy())
 print("Test data shapes:")
 for image, idnum in get_test_dataset().take(3):
print(image.numpy().shape, idnum.numpy().shape)
 print("Test data IDs:", idnum.numpy().astype('U'))<train_model>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
training_dataset = get_training_dataset()
training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )<define_variables>
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds))
display_batch_of_images(next(train_batch))<define_variables>
score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
test_dataset = get_test_dataset()
test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )<define_variables>
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
display_batch_of_images(next(test_batch))<choose_model_class>
score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<install_modules>
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
!pip install -q efficientnet 
<choose_model_class>
score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
with strategy.scope() :
enet = efn.EfficientNetB7(input_shape=[*IMAGE_SIZE, 3], weights='noisy-student', include_top=False)
 enet.trainable = True
model1 = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model1.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model1.summary()<train_model>
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
if not SKIP_VALIDATION:
history1 = model1.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
validation_data=get_validation_dataset() ,
callbacks = [lr_callback])
 else:
history1 = model1.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback] )<choose_model_class>
score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
with strategy.scope() :
densenet = tf.keras.applications.DenseNet201(input_shape=[*IMAGE_SIZE, 3], weights='imagenet', include_top=False)
 densenet.trainable = True
model2 = tf.keras.Sequential([
densenet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()<train_model>
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42 )
if not SKIP_VALIDATION:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
validation_data=get_validation_dataset() ,
callbacks = [lr_callback])
 else:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback] )<predict_on_test>
score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m1 = model1.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m1+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
best_alpha = np.argmax(scores)/100
else:
best_alpha = 0.51
print('Best alpha: ' + str(best_alpha))<compute_test_metric>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)
if not SKIP_VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<predict_on_test>
score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
def predict_tta(model, n_iter):
probs = []
for i in range(n_iter):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs.append(model.predict(test_images_ds,verbose=0))
return probs<predict_on_test>
xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )
test_ds = get_test_dataset(ordered=True)
 print('Calculating predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs1 = np.mean(predict_tta(model1, TTA_NUM), axis=0)
 probs2 = np.mean(predict_tta(model2, TTA_NUM), axis=0)
 probabilities = best_alpha*probs1 +(1-best_alpha)*probs2
predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
!pip install -q efficientnet<set_options>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )
%matplotlib inline
print("Tensorflow version " + tf.__version__ )<define_variables>
print('START Fit')
 print(datetime.now() , 'StackingCVRegressor')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )
GCS_DS_PATH = KaggleDatasets().get_gcs_path("tpu-getting-started" )<define_variables>
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.25 * stack_gen_model.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )
IMAGE_SIZE = [512, 512]
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SEED = 2020<data_type_conversions>
submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)) )
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=True)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print('Dataset: {} validation images, {} unlabeled test images'.format(NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1 )
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
with strategy.scope() :
model1 = get_model(DenseNet201)
 model1.load_weights("/kaggle/input/start-with-pre-train-0895d8/my_ef_net_b7.h5" )<choose_model_class>
submission.to_csv("new_submission.csv", index=False)
 print("Submission was successfully saved!" )