Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
%matplotlib inline
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 enet.trainable = True
model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax', dtype='float32')
 ] )<choose_model_class>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy']
)<choose_model_class>
train["SalePrice"] = np.log1p(train["SalePrice"] )
callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=2, verbose=1),
tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1, restore_best_weights=True),
]<train_model>
data = pd.concat([train, test] )
history = model.fit(
 train_data,
validation_data = val_data,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = callbacks,
)<categorify>
data.set_index('Id', inplace = True )
def read_unlabeled_tfrecord(example):
tfrec_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum<define_variables>
data.isnull().sum() [:50]
test_data = tf.data.TFRecordDataset(
 tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-' + str(IMAGE_SIZE)+ 'x' + str(IMAGE_SIZE)+ '/test/*.tfrec'),
num_parallel_reads = tf.data.experimental.AUTOTUNE 
)
 test_data = test_data.with_options(tf.data.Options())
 test_data = test_data.map(read_unlabeled_tfrecord, num_parallel_calls = tf.data.experimental.AUTOTUNE)
 test_data = test_data.batch(BATCH_SIZE)
 test_data = test_data.prefetch(tf.data.experimental.AUTOTUNE )<predict_on_test>
data['PoolQC'].fillna('None', inplace=True)
 data['MiscFeature'].fillna('None', inplace=True)
 data['Alley'].fillna('None', inplace=True)
 data['Fence'].fillna('None', inplace=True)
 data['FireplaceQu'].fillna('None', inplace=True )
test_images = test_data.map(lambda image, idnum: image)
 probabilities = model.predict(test_images)
 predictions = np.argmax(probabilities, axis=-1 )<data_type_conversions>
data["LotFrontage"] = data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
ids = []
for image, image_ids in test_data.take(NUM_TEST_IMAGES):
ids.append(image_ids.numpy())
 ids = np.concatenate(ids, axis=None ).astype(str )<save_to_csv>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
data[col] = data[col].fillna('None' )
submission = pd.DataFrame(data={'id': ids, 'label': predictions})
 submission.to_csv('submission.csv', index=False )<save_model>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0 )
model.save('model.h5' )<install_modules>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0 )
!pip install -q transformers==3.0.2
!pip install -q nlp<import_modules>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None' )
np.random.seed(12345 )<set_options>
data["MasVnrType"] = data["MasVnrType"].fillna("None")
 data["MasVnrArea"] = data["MasVnrArea"].fillna(0 )
pd.set_option('display.max_colwidth', 100)
 os.environ["WANDB_API_KEY"] = "0"
sns.set_context("talk", font_scale=1.05 )<load_from_csv>
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0] )
train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv" )<count_unique_values>
data = data.drop(['Utilities'], axis=1 )
print("Any duplicate rows(train or test): ",
max(train['id'].nunique() != train.shape[0], test['id'].nunique() != test.shape[0]))<merge>
data["Functional"] = data["Functional"].fillna("Typ" )
print("Train and test datasets have unique, non-overlapping ids: ",
pd.merge(train['id'], test['id'], on = 'id', how = 'inner' ).shape[0] == 0 )<count_values>
data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0] )
print("% distribution by language - training dataset")
 train['language'].value_counts(normalize=True)* 100 .<count_values>
data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0] )
print("% distribution by language - test dataset")
 test['language'].value_counts(normalize=True)* 100 .<categorify>
data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0] )
train['label_str'] = train['label'].map({0 : "entailment", 1 : "neutral", 2 : "contradiction"} )<string_transform>
data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0] )
def _get_word_count(snt):
return len(str(snt ).split() )<feature_engineering>
data['MSSubClass'] = data['MSSubClass'].fillna("None" )
train['premise_len'] = train['premise'].apply(lambda x: _get_word_count(x))
train['hypothesis_len'] = train['hypothesis'].apply(lambda x: _get_word_count(x))
train['relative_diff'] =(train['hypothesis_len'] - train['premise_len'])* 1./ train['premise_len']<create_dataframe>
data['MSSubClass'] = data['MSSubClass'].apply(str)
 data['OverallCond'] = data['OverallCond'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str )
snli = load_dataset(path='snli' )<create_dataframe>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(data[c].values))
data[c] = lbl.transform(list(data[c].values))
print('Shape data: {}'.format(data.shape))
result_snli = []
for k in ['train', 'validation']:
for record in snli[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result_snli.append(( c1,c2,'en','English',c3))
snli_df = pd.DataFrame(result_snli, columns=['premise','hypothesis','lang_abv', 'language', 'label'] )<merge>
data = pd.get_dummies(data)
 print(data.shape )
pd.merge(train['premise'], snli_df['premise'], on = 'premise', how='inner' ).shape[0] != 0<drop_column>
train = data[:1460]
test = data[1460:].drop('SalePrice', axis = 1 )
final_df = shuffle(train.drop(columns=['id', 'label_str', 'premise_len', 'hypothesis_len', 'relative_diff'])).reset_index(drop = True )<split>
X = train.drop('SalePrice', axis = 1)
 y = train['SalePrice']
X, y = final_df[['premise', 'hypothesis']].values.tolist() , final_df['label']
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25, random_state=12345 )<set_options>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42 )
del snli
gc.collect()<define_variables>
xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1,colsample_bytree=0.6,
gamma=0, importance_type='gain',
learning_rate=0.01, max_delta_step=0,
max_depth=4,min_child_weight=1.5,
n_estimators=2500, n_jobs=1, nthread=None,
objective='reg:linear', reg_alpha=0.4640,
reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1 )
encoder_handle = 'jplu/tf-xlm-roberta-large'<load_pretrained>
lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=11000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4 )
tokenizer = AutoTokenizer.from_pretrained(encoder_handle )<init_hyperparams>
gboost = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state =5 )
max_len = 64
random_seed = 11887
learning_rate = 1e-5
epochs = 5
batch_size = 16 * strategy.num_replicas_in_sync
loss = 'sparse_categorical_crossentropy'
metrics = ['accuracy']
steps_per_epoch = 1000
auto = tf.data.experimental.AUTOTUNE<categorify>
xgb.fit(X_train, y_train)
 lgbm.fit(X_train, y_train, eval_metric='rmsle')
 gboost.fit(X_train, y_train )
def encode_sentence(s, tokenizer):

 tokens = list(tokenizer.tokenize(s))
tokens.append(tokenizer.sep_token)
 return tokenizer.convert_tokens_to_ids(tokens)
 def tokenize(data, tokenizer, max_len):

 PAD_ID = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
 tokens1 = tf.ragged.constant([encode_sentence(s[0], tokenizer)for s in data], dtype=tf.int32)
 tokens2 = tf.ragged.constant([encode_sentence(s[1], tokenizer)for s in data], dtype=tf.int32)
 cls_label = [tokenizer.convert_tokens_to_ids([tokenizer.cls_token])]*tokens1.shape[0]
tokens = tf.concat([cls_label, tokens1, tokens2], axis=-1)
 tokens = tokens[:, :max_len]
tokens = tokens.to_tensor(default_value=PAD_ID)
 pad = max_len - tf.shape(tokens)[1]
tokens = tf.pad(tokens, [[0, 0], [0, pad]], constant_values=PAD_ID)
 input_word_ids = tf.reshape(tokens, [-1, max_len])
 input_mask = tf.cast(input_word_ids != PAD_ID, tf.int32)
 input_mask = tf.reshape(input_mask, [-1, max_len])
 input_type_ids = tf.concat([tf.zeros_like(cls_label), tf.zeros_like(tokens1), tf.ones_like(tokens2)], axis=-1 ).to_tensor()
inputs = {
'input_word_ids': input_word_ids,
'input_mask': input_mask,
'input_type_ids': input_type_ids}
return inputs<train_model>
pred2 = xgb.predict(X_test)
 pred3 = lgbm.predict(X_test)
 pred4 = gboost.predict(X_test )
def build_dataset(x, y, mode, batch_size):

 if mode == "train":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.repeat() 
.shuffle(5678)
.batch(batch_size)
.prefetch(auto)
)
 elif mode == "valid":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 elif mode == "test":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(x)
.batch(batch_size)
)
 else:
raise NotImplementedError
return dataset<string_transform>
print('Erro médio logarítmico de raiz quadrado test(XGB)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred2))))
print('Erro médio logarítmico de raiz quadrado test(LGBM)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred3))))
print('Erro médio logarítmico de raiz quadrado test(GBoost)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred4))))
x_train_ = tokenize(x_train, tokenizer, max_len)
 x_valid_ = tokenize(x_valid, tokenizer, max_len )<create_dataframe>
lgbm.fit(X, y)
 xgb.fit(X ,y)
 gboost.fit(X, y )
train_dataset = build_dataset(x_train_, y_train, "train", batch_size)
 valid_dataset = build_dataset(x_valid_, y_valid, "valid", batch_size )<choose_model_class>
prediction_lgbm = np.expm1(lgbm.predict(test))
prediction_xgb = np.expm1(xgb.predict(test))
prediction_gboost = np.expm1(gboost.predict(test))
def build_model(encoder_handle, random_seed, learning_rate, loss, metrics, max_len):
tf.keras.backend.clear_session()
tf.random.set_seed(random_seed)
 with strategy.scope() :
input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 roberta = TFXLMRobertaModel.from_pretrained(encoder_handle)
 roberta = roberta([input_word_ids, input_mask])[0]
out = GlobalAveragePooling1D()(roberta)
 out = Dense(3, activation='softmax' )(out)
 model = Model(inputs=[input_word_ids, input_mask], outputs = out)
 model.compile(optimizer=Adam(lr=learning_rate), loss=loss, metrics=metrics)
 model.summary()
return model<choose_model_class>
model = build_model(encoder_handle, random_seed, learning_rate, loss, metrics, max_len )<choose_model_class>
prediction =(prediction_lgbm * 0.38 + prediction_gboost * 0.36 + prediction_xgb * 0.26 )
early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
verbose=1,
patience=2,
mode='min',
restore_best_weights=True )<prepare_x_and_y>
submission = pd.DataFrame({"Id": test.index,"SalePrice": prediction} )
x_small_train, y_small_train = X[:batch_size], y[:batch_size]<create_dataframe>
submission.to_csv('submission.csv', index=False )