Kaggle competition: House Prices - Advanced Regression Techniques
train_dups = dups[(dups.col1t == 'Train')&(dups.col1t == dups.col2t)].merge(train, left_on='col1', right_on='image_id' ).merge(train, left_on='col2', right_on='image_id')
 train_dups.head()<filter>
categorical = pd.read_csv("/kaggle/input/categorical-raw/full_features.csv")
 features = pd.read_csv("/kaggle/input/housepriceregression-featurecreation/FeatureFilled.csv")
 targets = pd.read_csv("/kaggle/input/housepriceregression-featurecreation/salePrice.csv")
 with open("/kaggle/input/housepriceregression-featurecreation/new_adboost", "rb")as file:
new_adboost = pickle.load(file )
confused = train_dups[train_dups.label_x != train_dups.label_y]
confused<randomize_order>
features_sub = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1 )
print(f'list of duplicates train / train: {len(train_dups)}')
 print(len(train))
train = train[~train.image_id.isin(list(train_dups['col2'])) ].reset_index(drop=True)
 print(len(train))
train = train[~train.image_id.isin(list(confused['col1'])) ].reset_index(drop=True)
 train = train[~train.image_id.isin(list(confused['col2'])) ].reset_index(drop=True)
 print(len(train))<filter>
features_add = features.copy()
features_add['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features_add['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features_add['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features_add['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features_add['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'] )
dups[(dups.col1t == 'Test')&(dups.col1t == dups.col2t)]<merge>
features_add['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features_add['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features_add['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features_add['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features_add['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
solutions = dups[dups.col1t != dups.col2t].merge(train, left_on='col2', right_on='image_id' ).sort_values(by='label')
 solutions.head()<split>
features_addsub = features_add.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features_addsub
skf = StratifiedKFold(shuffle=True, random_state=42)
 for i,(train_index, val_index)in enumerate(skf.split(train, train['label'])) :
print(train_index.shape, val_index.shape)
 train.loc[val_index, 'fold'] = i<define_variables>
features_pure_log = features_addsub.apply(lambda x: np.log1p(x))
features_pure_log
train = train.iloc[list(train.index)+ [x for x in train[train.label == 'multiple_diseases'].index for j in range(0,2)]]
train = train.reset_index(drop=True )<define_variables>
train = features_pure_log.iloc[:1460]
train1 = features_pure_log.iloc[:1460]
folds = range(0,3)
<init_hyperparams>
X_train, X_test, y_train, y_test = train_test_split(train1, target, test_size=0.5, random_state=21 )
R1=(400,600)
 R2=(320,512)
 TTA=True
EPOCH1=15
EPOCH2=-1
LR1=5e-2
LR2=slice(1e-7, 1e-4)
 BS=16
tfms = [Rotate(max_deg=30),
Zoom() ,
Brightness() ,
Flip() ,
Contrast() ,
Resize(R2),
Normalize.from_stats(*imagenet_stats)]
comp = setup_aug_tfms(tfms)
 for fold in folds:
plant = DataBlock(blocks=(ImageBlock, CategoryBlock),
splitter=IndexSplitter(train[train['fold'] == fold].index),
get_x=ColReader('image_id', pref=path/"images", suff='.jpg'),
get_y=ColReader('label'),
item_tfms=Resize(R1),
batch_tfms=comp)
 dls = plant.dataloaders(train, bs=BS, seed=42)
 learn = cnn_learner(dls,
model_f,
pretrained=True,
loss_func=LabelSmoothingCrossEntropy() ,
cbs=SaveModelCallback(monitor='roc_auc_score', comp=np.greater),
model_dir='/kaggle/working',
metrics=[RocAuc() , error_rate])
 print(f'-------- {fold} ---------')
 if EPOCH1 > 0:
learn.fine_tune(EPOCH1,
base_lr=LR1,
cbs=SaveModelCallback(monitor='roc_auc_score', comp=np.greater))
learn.recorder.plot_loss()
if EPOCH2 > 0:
learn.unfreeze()
learn.fit_one_cycle(EPOCH2,
lr_max=LR2,
cbs=SaveModelCallback(monitor='roc_auc_score', comp=np.greater))
learn.recorder.plot_loss()
learn.save(f'seresnext50-fold-{fold}')
 tst_dl = dls.test_dl(test)
 if TTA:
preds, _ = learn.tta(dl=tst_dl)
 else:
preds, _ = learn.get_preds(dl=tst_dl)
 cols=['image_id'] + list(dls.vocab)
 res = pd.concat([test, pd.DataFrame(preds,columns = learn.dls.vocab)],axis=1)[cols]
res.to_csv(f'submission_resnext_{fold}.csv',index=False)
 print('confident multiple_diseases')
 my_ids = res[res['multiple_diseases'] > 0.9].sort_values(by='multiple_diseases' ).image_id.values
print(my_ids)
 res = learn.get_preds()
cols= dls.vocab
r = pd.DataFrame(res[0], columns=cols)
 valid_with_preds = pd.concat([train[train['fold'] == fold].reset_index(drop=True), r], axis=1)
 print(roc_auc_score(train[train['fold'] == fold][cols], r))
for col in cols:
score = roc_auc_score(train[train['fold'] == fold][col], r[col])
 print(f'{col} -> {score}')
 valid_with_preds.to_csv(f'validation_resnext_{fold}.csv',index=False )<feature_engineering>
from sklearn.preprocessing import RobustScaler, StandardScaler, MinMaxScaler
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.svm import SVR
from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor, StackingRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.model_selection import KFold, cross_val_score
from sklearn.pipeline import make_pipeline
if len(folds)> 1:
my_df = {}
for fold in folds:
my_df[fold] = pd.read_csv(f'submission_resnext_{fold}.csv' ).set_index('image_id')
 for x in ['healthy','multiple_diseases','rust','scab']:
for fold in folds:
my_df[fold][x] = rankdata(my_df[fold][x], method='min')
 for x in ['healthy','multiple_diseases','rust','scab']:
for fold in folds[1:]:
my_df[folds[0]][x] += my_df[fold][x]
my_df[folds[0]][x] = my_df[folds[0]][x] / 5
my_df[folds[0]].reset_index().to_csv('submission_resnext.csv', index=False)
 else:
print('no need for rankdata' )<import_modules>
def rmsle(estimator, x, y):
rmsle = np.sqrt(mean_squared_error(y, estimator.predict(x)))
 print(f"RMSLE score : {rmsle}" )
import pandas as pd<import_modules>
kfolds = KFold(n_splits=10, shuffle=True, random_state=12 )
import pandas as pd<load_from_csv>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=np.linspace(10, 200, 10),
scoring="neg_mean_squared_error", cv=kfolds))
ridge.fit(X_train, y_train)
 rmsle(ridge, X_test, y_test )
sub = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub1 = pd.read_csv('.. /input/plant-best-sub/resources/submission1.csv')
 sub2 = pd.read_csv('.. /input/plant-best-sub/resources/submission2.csv')
 sub3 = pd.read_csv('.. /input/plant-best-sub/resources/submission3.csv')
 sub.healthy =(sub1.healthy + sub2.healthy + sub3.healthy)/3
sub.multiple_diseases =(sub1.multiple_diseases + sub2.multiple_diseases + sub3.multiple_diseases)/3
sub.rust =(sub1.rust + sub2.rust + sub3.rust)/3
sub.scab =(sub1.scab + sub2.scab + sub3.scab)/3<save_to_csv>
lasso = make_pipeline(RobustScaler() , LassoCV(cv=kfolds, max_iter=1e7, random_state=23))
lasso.fit(X_train, y_train)
 rmsle(lasso, X_test, y_test )
sub.to_csv('submission.csv', index=False )<save_to_csv>
elastic = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, cv=kfolds, random_state=32))
elastic.fit(X_train, y_train)
 rmsle(elastic, X_test, y_test )
sub.to_csv('submission.csv', index=False )<install_modules>
svr = make_pipeline(RobustScaler() , SVR(kernel='poly' , C=1., degree=1,
gamma='auto', epsilon=0.001))
svr.fit(X_train, y_train)
 rmsle(svr, X_test, y_test )
!pip install -q efficientnet<import_modules>
gbr = GradientBoostingRegressor(n_estimators=4000, learning_rate=0.01, max_depth=3,
subsample=0.9, max_features='sqrt' , random_state=21)
 gbr.fit(X_train, y_train)
 rmsle(gbr, X_test, y_test )
%matplotlib inline
print("Tensorflow version " + tf.__version__ )<choose_model_class>
xgboost = XGBRegressor(n_estimators=4000, learning_rate=0.009, max_depth=4,
min_child_weight=2., colsample_bytree=0.8 , subsample=0.69,
gamma=0.01, reg_lambda=1.1, reg_alpha=0.001, random_state=21)
 xgboost.fit(X_train, y_train)
 rmsle(xgboost, X_test, y_test )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path("plant-pathology-2020-fgvc7")
 EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync<define_variables>
lgbm = LGBMRegressor(n_estimators=1000, learning_rate=0.04, num_leaves=8, max_depth=3,
colsample_bytree=0.1, objective='regression',
random_state=21)
 lgbm.fit(X_train, y_train)
 rmsle(lgbm, X_test, y_test )
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
estimators = [ridge, lasso, elastic, svr, gbr, lgbm, xgboost]
names = ["RidgeCV", "LassoCV", "ElasticNetCV", "SVR", "GBoost", "LGBM", "XGBoost"]
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.15, random_state=2020 )<normalization>
dtr = DecisionTreeRegressor(max_depth=10, max_features='sqrt', min_samples_leaf=2)
 ada = AdaBoostRegressor(xgboost, learning_rate=0.1, n_estimators=10, loss='exponential', random_state=11)
 ada.fit(X_train, y_train)
 rmsle(ada, X_test, y_test )
img_size = 768
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
features_pure_log.to_csv("FeaturesLog.csv")
 target_log.to_csv("TargetLog.csv" )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<choose_model_class>
ridge.fit(train1, target)
 lasso.fit(train1, target)
 elastic.fit(train1, target)
 svr.fit(train1, target)
 gbr.fit(train1, target)
 lgbm.fit(train1, target)
 xgboost.fit(train1, target)
 ada.fit(train1, target )
def get_model(use_model):
base_model = use_model(weights='noisy-student',
include_top=False, pooling='avg',
input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(train_labels.shape[1], activation="softmax" )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(optimizer='nadam', loss='categorical_crossentropy',metrics=['categorical_accuracy'])
 return model
with strategy.scope() :
model1 = get_model(EfficientNetB7)
 model1.load_weights("/kaggle/input/tf-zoo-models-on-tpu-efficientnetb7/my_ef_net_b7.h5")
<choose_model_class>
test = features_pure_log.iloc[1460:]
test
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(train_labels.shape[1], activation="softmax" )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(optimizer='nadam', loss='categorical_crossentropy',metrics=['categorical_accuracy'])
 return model
with strategy.scope() :
model2 = get_model(DenseNet201)
 model2.load_weights("/kaggle/input/tf-zoo-models-on-tpu-densenet201/my_dense_net_201.h5")
<choose_model_class>
def blend_models(X):
return(0.1 * ridge.predict(X)) +(0.1 * lasso.predict(X)) +(0.1 * elastic.predict(X)) +\ 
(0.05 * svr.predict(X)) +(0.15 * gbr.predict(X)) +(0.1 * lgbm.predict(X)) +\ 
(0.3 * xgboost.predict(X)) +(0.1 * ada.predict(X))
prediction = np.expm1(blend_models(test))
prediction
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(train_labels.shape[1], activation="softmax" )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(optimizer='nadam', loss='categorical_crossentropy',metrics=['categorical_accuracy'])
 return model
with strategy.scope() :
model3 = get_model(InceptionResNetV2)
 model3.load_weights("/kaggle/input/tf-zoo-models-on-tpu/InceptionResNetV2.h5")
<save_to_csv>
submit = pd.DataFrame({"Id": np.arange(1461, 2920), "SalePrice": prediction})
 submit.to_csv("submission12a.csv", index=False )