Kaggle competition: House Prices - Advanced Regression Techniques
if not SKIP_VALIDATION:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
validation_data=get_validation_dataset() ,
callbacks = [lr_callback])
 else:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback] )<predict_on_test>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 train
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m1 = model1.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m1+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
best_alpha = np.argmax(scores)/100
else:
best_alpha = 0.51
print('Best alpha: ' + str(best_alpha))<compute_test_metric>
num_col = train.loc[:,'MSSubClass':'SaleCondition'].select_dtypes(exclude=['object'] ).columns
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2, num_col)
 train.loc[Outliers_to_drop]
if not SKIP_VALIDATION:
cm_probabilities = best_alpha*m1 +(1-best_alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<predict_on_test>
train = train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
def predict_tta(model, n_iter):
probs = []
for i in range(n_iter):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs.append(model.predict(test_images_ds,verbose=0))
return probs<predict_on_test>
test_id = test.Id
train_target = train.SalePrice
test_ds = get_test_dataset(ordered=True)
 print('Calculating predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs1 = np.mean(predict_tta(model1, TTA_NUM), axis=0)
 probs2 = np.mean(predict_tta(model2, TTA_NUM), axis=0)
 probabilities = best_alpha*probs1 +(1-best_alpha)*probs2
predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<import_modules>
df = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'], test.loc[:,'MSSubClass':'SaleCondition']))
print('Tensorflow version ' + tf.__version__)
 !pip install -q efficientnet 
<set_options>
def basic_details(df):
b = pd.DataFrame()
b['Missing value, %'] = round(df.isnull().sum() /df.shape[0]*100)
 b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
for col in df:
if df[col].dtype == 'object':
df[col].fillna('N', inplace=True)
 else: df[col].fillna(df[col].median() , inplace=True )
AUTO = tf.data.experimental.AUTOTUNE
IMAGE_SIZE = [224, 224]
EPOCHS = 30
FOLDS = 3
SEED = 777
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<set_options>
columns = [i for i in df.columns]
dummies = pd.get_dummies(df,columns=columns, drop_first=True,sparse=True )
MIXED_PRECISION = False
XLA_ACCELERATE = False
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )<categorify>
train = dummies.iloc[:train.shape[0], :]
test = dummies.iloc[train.shape[0]:, :]
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset,do_aug=True):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<normalization>
X = train
y = train_target
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label 
<create_dataframe>
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=10)
 d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(test )
train_dataset_all = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES})['TRAINING_FILENAMES']), labeled = True)
 test_dataset_all = load_dataset(list(pd.DataFrame({'TEST_FILENAMES': TEST_FILENAMES})['TEST_FILENAMES']), labeled = True, ordered=True)
 train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES_ONLY': TRAINING_FILENAMES_ONLY})['TRAINING_FILENAMES_ONLY']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'VAL_FILENAMES_ONLY': VAL_FILENAMES_ONLY})['VAL_FILENAMES_ONLY']), labeled=True, ordered=True)
 train_all = get_training_dataset(train_dataset_all)
 test = get_test_dataset(test_dataset_all)
 test_images_ds = test.map(lambda image, idnum: image)
 train = get_training_dataset(train_dataset)
 val = get_validation_dataset(val_dataset)
 def Xception_model() :
with strategy.scope() :
rnet = Xception(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def VGG16_model() :
with strategy.scope() :
rnet = VGG16(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def DenseNet201_model() :
with strategy.scope() :
rnet = DenseNet201(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def InceptionV3_model() :
with strategy.scope() :
rnet = InceptionV3(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def EfficientNetB7_model() :
with strategy.scope() :
rnet = EfficientNetB7(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def ResNet152V2_model() :
with strategy.scope() :
rnet = ResNet152V2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def MobileNetV2_model() :
with strategy.scope() :
rnet = MobileNetV2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def InceptionResNetV2_model() :
with strategy.scope() :
rnet = InceptionResNetV2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
models = {'Xception' : Xception_model,
'VGG16' : VGG16_model,
'DenseNet201' : DenseNet201_model,
'InceptionV3' : InceptionV3_model,
'InceptionResNetV2' : InceptionResNetV2_model
}
historys = {}
predictions = {}
predictions_val = {}
predictions_prob = {}
times = {}
MODELS_NUMBER = 5 
<prepare_output>
params = {
'objective':'reg:linear',
'n_estimators': 1000,
'booster':'gbtree',
'max_depth':4,
'eval_metric':'mae',
'learning_rate':0.05,
'min_child_weight':1,
'subsample':0.7,
'colsample_bytree':0.81,
'seed':45,
'reg_alpha':1e-05,
'gamma':0,
'nthread':-1
}
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 5000, watchlist, early_stopping_rounds=300, maximize=False, verbose_eval=10)
 p_test = clf.predict(d_test )
df = pd.DataFrame(predictions)
 pred = []
for i in range(0, 7382):
if df.loc[i,:].unique().shape[0] < MODELS_NUMBER :
pred.append(stats.mode(df.loc[i,:].values)[0][0])
 else :
pred.append(df.loc[i,'InceptionResNetV2'])
 df = pd.DataFrame(predictions_val)
 pred_val = []
for i in range(0, 3712):
if df.loc[i,:].unique().shape[0] < MODELS_NUMBER :
pred_val.append(stats.mode(df.loc[i,:].values)[0][0])
 else :
pred_val.append(df.loc[i,'InceptionResNetV2'])
 avg_prob = predictions_prob['Xception'] + predictions_prob['VGG16'] + predictions_prob['DenseNet201'] + predictions_prob['InceptionV3'] + predictions_prob['InceptionResNetV2']
pred_avg = pd.DataFrame(np.argmax(avg_prob, axis=-1))<save_to_csv>
leaks = {
1461:105000,
1477:290941,
1492:67500,
1494:362500,
1514:84900,
1521:108538,
1531:80400,
1537:12789,
1540:76500,
1545:134000,
1554:122000,
1556:107500,
1557:100000,
1559:93369,
1560:114900,
1566:270000,
1567:85000,
1572:128000,
1573:308030,
1575:270000,
1586:84900,
1587:155891,
1589:64000,
1595:100000,
1597:215000,
1603:50138,
1610:174000,
1611:169000,
1615:76000,
1616:88250,
1617:85500,
1620:159000,
1622:161000,
1631:240000,
1638:154000,
1650:76500,
1652:111000,
1661:462000,
1664:610000,
1666:296000,
1678:552000,
1696:245000,
1698:327000,
1712:264500,
1717:152000,
1720:203000,
1726:171500,
1727:145000,
1737:275000,
1767:256000,
1774:135000,
1786:142900,
1787:156500,
1788:59000,
1790:78500,
1793:163000,
1807:103500,
1814:80000,
1820:58500,
1823:44000,
1831:179900,
1832:62500,
1835:97500,
1837:70000,
1842:63000,
1843:113500,
1863:269500,
1864:269500,
1892:85000,
1895:103500,
1912:315000,
1913:123000,
1915:230000,
1916:57625,
1925:170000,
1946:115000,
1947:334000,
1967:317500,
1970:390000,
1971:460000,
1975:615000,
1976:284000,
1996:284500,
1997:291000,
2004:297900,
2014:163000,
2030:300000,
2031:285000,
2032:290000,
2033:305000,
2038:345000,
2052:140000,
2055:141500,
2068:146000,
2076:94000,
2086:143000,
2093:122250,
2099:46500,
2100:65000,
2101:139500,
2106:55000,
2107:184000,
2111:108000,
2152:260000,
2162:475000,
2163:395039,
2180:185000,
2185:165000,
2206:104000,
2207:257076,
2208:263400,
2211:126000,
2217:13100,
2220:65000,
2223:300000,
2227:241500,
2230:172500,
2232:150000,
2235:195000,
2236:298751,
2238:209200,
2239:146000,
2245:94900,
2251:103000,
2263:349265,
2264:591587,
2267:441929,
2268:455000,
2269:174000,
2288:322400,
2295:500067,
2342:260000,
2354:146000,
2362:300000,
2375:279700,
2376:255000,
2379:240050,
2380:162500,
2395:224500,
2404:175000,
2419:115000,
2437:125500,
2455:136500,
2461:132000,
2465:165000,
2466:90000,
2468:113000,
2469:117000,
2474:50000,
2495:109900,
2544:110000,
2550:183850,
2557:79275,
2564:238000,
2565:153500,
2572:200000,
2574:315000,
2583:375000,
2590:244000,
2591:257000,
2599:392000,
2610:138000,
2611:80000,
2617:169000,
2618:252000,
2627:130000,
2631:535000,
2632:401179,
2634:470000,
2638:294323,
2658:344133,
2673:246990,
2690:405749,
2702:129500,
2723:157500,
2741:132000,
2752:167000,
2754:180000,
2760:80000,
2775:111500,
2776:156500,
2779:111500,
2788:64000,
2793:202500,
2794:75000,
2805:125000,
2813:156500,
2823:415000,
2829:224500,
2832:233555,
2859:98000,
2866:134000,
2872:35000,
2873:121000,
2881:195000,
2916:71000,
2917:131000,
2919:188000
}
test_ids_ds = test.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission_vote.csv', np.rec.fromarrays([test_ids, pred]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, np.argmax(avg_prob, axis=-1)]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] = p_test
sub['SalePrice'] = sub.apply(lambda r: leaks[int(r['Id'])] if int(r['Id'])in leaks else r['SalePrice'], axis=1)
 sub.to_csv('submission.csv', index=False )