Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<create_dataframe>
warnings.simplefilter("ignore")
 sns.set()
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=False, batch_size=16, dim=512):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTOTUNE)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTOTUNE)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTOTUNE)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTOTUNE)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTOTUNE)
 return ds<define_variables>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 sample = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv" )
class DataGenerator(Sequence):
def __init__(self, path, list_IDs, labels, batch_size, img_size, img_channel):
self.path = path
self.list_IDs = list_IDs
self.labels = labels
self.batch_size = batch_size
self.img_size = img_size
self.img_channel = img_channel
self.indexes = np.arange(len(self.list_IDs))
def __len__(self):
return int(np.floor(len(self.list_IDs)/self.batch_size))
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
X, y = self.__data_generation(list_IDs_temp)
 return X, y
def __data_generation(self, list_IDs_temp):
X = np.empty(( self.batch_size, self.img_size, self.img_size, self.img_channel))
y = np.empty(( self.batch_size, 5), dtype=int)
 for i, ID in enumerate(list_IDs_temp):
data_file = cv2.imread(self.path+ID)
 img = cv2.resize(data_file,(self.img_size, self.img_size))
X[i, ] = img
y[i, ] = self.labels[i]
X = X.astype('float32')
 X /= 255.
 return X, y<prepare_x_and_y>
train.drop("Id", axis=1, inplace=True)
 test.drop("Id", axis=1, inplace=True )
<define_variables>
target = train["SalePrice"]
train = train.drop("SalePrice", axis=1 )
test_generator = DataGenerator('.. /input/cassava-leaf-disease-classification/'+'test_images/', test_df['image_id'], test_df['label'], 1, DIM, 3 )<import_modules>
train.isnull().sum().sum()
BASE_WEIGHTS_PATH = 'https://storage.googleapis.com/keras-applications/'
WEIGHTS_HASHES = {
'b0':('902e53a9f72be733fc0bcb005b3ebbac',
'50bc09e76180e00e4465e1a485ddc09d'),
'b1':('1d254153d4ab51201f1646940f018540',
'74c4e6b3e1f6a1eea24c589628592432'),
'b2':('b15cce36ff4dcbd00b6dd88e7857a6ad',
'111f8e2ac8aa800a7a99e3239f7bfb39'),
'b3':('ffd1fdc53d0ce67064dc6a9c7960ede0',
'af6d107764bb5b1abb91932881670226'),
'b4':('18c95ad55216b8f92d7e70b3a046e2fc',
'ebc24e6d6c33eaebbd558eafbeedf1ba'),
'b5':('ace28f2a6363774853a83a0b21b9421a',
'38879255a25d3c92d5e44e04ae6cec6f'),
'b6':('165f6e37dce68623721b423839de8be5',
'9ecce42647a20130c1f39a5d4cb75743'),
'b7':('8c03f828fec3ef71311cd463b6759d99',
'cbcfe4450ddf6f3ad90b1b398090fe4a'),
}
DEFAULT_BLOCKS_ARGS = [{
'kernel_size': 3,
'repeats': 1,
'filters_in': 32,
'filters_out': 16,
'expand_ratio': 1,
'id_skip': True,
'strides': 1,
'se_ratio': 0.25
}, {
'kernel_size': 3,
'repeats': 2,
'filters_in': 16,
'filters_out': 24,
'expand_ratio': 6,
'id_skip': True,
'strides': 2,
'se_ratio': 0.25
}, {
'kernel_size': 5,
'repeats': 2,
'filters_in': 24,
'filters_out': 40,
'expand_ratio': 6,
'id_skip': True,
'strides': 2,
'se_ratio': 0.25
}, {
'kernel_size': 3,
'repeats': 3,
'filters_in': 40,
'filters_out': 80,
'expand_ratio': 6,
'id_skip': True,
'strides': 2,
'se_ratio': 0.25
}, {
'kernel_size': 5,
'repeats': 3,
'filters_in': 80,
'filters_out': 112,
'expand_ratio': 6,
'id_skip': True,
'strides': 1,
'se_ratio': 0.25
}, {
'kernel_size': 5,
'repeats': 4,
'filters_in': 112,
'filters_out': 192,
'expand_ratio': 6,
'id_skip': True,
'strides': 2,
'se_ratio': 0.25
}, {
'kernel_size': 3,
'repeats': 1,
'filters_in': 192,
'filters_out': 320,
'expand_ratio': 6,
'id_skip': True,
'strides': 1,
'se_ratio': 0.25
}]
CONV_KERNEL_INITIALIZER = {
'class_name': 'VarianceScaling',
'config': {
'scale': 2.0,
'mode': 'fan_out',
'distribution': 'truncated_normal'
}
}
DENSE_KERNEL_INITIALIZER = {
'class_name': 'VarianceScaling',
'config': {
'scale': 1./ 3.,
'mode': 'fan_out',
'distribution': 'uniform'
}
}
def EfficientNet(
 width_coefficient,
depth_coefficient,
default_size,
dropout_rate=0.2,
drop_connect_rate=0.2,
depth_divisor=8,
activation='swish',
blocks_args='default',
model_name='efficientnet',
include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
classifier_activation='softmax', 
):

 if blocks_args == 'default':
blocks_args = DEFAULT_BLOCKS_ARGS
if not(weights in {'imagenet', None} or os.path.exists(weights)) :
raise ValueError('The `weights` argument should be either '
'`None`(random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
 if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
 input_shape = imagenet_utils.obtain_input_shape(
 input_shape,
default_size=default_size,
min_size=32,
data_format=backend.image_data_format() ,
require_flatten=include_top,
weights=weights)
 if input_tensor is None:
img_input = layers.Input(shape=input_shape)
 else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
 else:
img_input = input_tensor
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
def round_filters(filters, divisor=depth_divisor):

 filters *= width_coefficient
new_filters = max(divisor, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats):

 return int(math.ceil(depth_coefficient * repeats))
x = img_input
x = layers.Normalization(axis=bn_axis )(x)
 x = layers.ZeroPadding2D(
 padding=imagenet_utils.correct_pad(x, 3),
name='stem_conv_pad' )(x)
 x = layers.Conv2D(
 round_filters(32),
3,
strides=2,
padding='valid',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name='stem_conv' )(x)
 x = layers.BatchNormalization(axis=bn_axis, name='stem_bn' )(x)
 x = layers.Activation(activation, name='stem_activation' )(x)
 blocks_args = copy.deepcopy(blocks_args)
 b = 0
blocks = float(sum(args['repeats'] for args in blocks_args))
for(i, args)in enumerate(blocks_args):
assert args['repeats'] > 0
args['filters_in'] = round_filters(args['filters_in'])
 args['filters_out'] = round_filters(args['filters_out'])
 for j in range(round_repeats(args.pop('repeats'))):
if j > 0:
args['strides'] = 1
args['filters_in'] = args['filters_out']
x = block(
 x,
activation,
drop_connect_rate * b / blocks,
name='block{}{}_'.format(i + 1, chr(j + 97)) ,
**args)
 b += 1
x = layers.Conv2D(
 round_filters(1280),
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name='top_conv' )(x)
 x = layers.BatchNormalization(axis=bn_axis, name='top_bn' )(x)
 x = layers.Activation(activation, name='top_activation' )(x)
 if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 if dropout_rate > 0:
x = layers.Dropout(dropout_rate, name='top_dropout' )(x)
 imagenet_utils.validate_activation(classifier_activation, weights)
 x = layers.Dense(
 classes,
activation=classifier_activation,
kernel_initializer=DENSE_KERNEL_INITIALIZER,
name='predictions' )(x)
 else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 elif pooling == 'max':
x = layers.GlobalMaxPooling2D(name='max_pool' )(x)
 if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
 else:
inputs = img_input
model = training.Model(inputs, x, name=model_name)
 if weights == 'imagenet':
if include_top:
file_suffix = '.h5'
file_hash = WEIGHTS_HASHES[model_name[-2:]][0]
else:
file_suffix = '_notop.h5'
file_hash = WEIGHTS_HASHES[model_name[-2:]][1]
file_name = model_name + file_suffix
weights_path = data_utils.get_file(
 file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
 model.load_weights(weights_path)
 elif weights is not None:
model.load_weights(weights)
 return model
def block(inputs,
activation='swish',
drop_rate=0.,
name='',
filters_in=32,
filters_out=16,
kernel_size=3,
strides=1,
expand_ratio=1,
se_ratio=0.,
id_skip=True):

 bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
filters = filters_in * expand_ratio
if expand_ratio != 1:
x = layers.Conv2D(
 filters,
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'expand_conv' )(
 inputs)
 x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn' )(x)
 x = layers.Activation(activation, name=name + 'expand_activation' )(x)
 else:
x = inputs
if strides == 2:
x = layers.ZeroPadding2D(
 padding=imagenet_utils.correct_pad(x, kernel_size),
name=name + 'dwconv_pad' )(x)
 conv_pad = 'valid'
else:
conv_pad = 'same'
x = layers.DepthwiseConv2D(
 kernel_size,
strides=strides,
padding=conv_pad,
use_bias=False,
depthwise_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'dwconv' )(x)
 x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn' )(x)
 x = layers.Activation(activation, name=name + 'activation' )(x)
 if 0 < se_ratio <= 1:
filters_se = max(1, int(filters_in * se_ratio))
se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze' )(x)
 se = layers.Reshape(( 1, 1, filters), name=name + 'se_reshape' )(se)
 se = layers.Conv2D(
 filters_se,
1,
padding='same',
activation=activation,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'se_reduce' )(
 se)
 se = layers.Conv2D(
 filters,
1,
padding='same',
activation='sigmoid',
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'se_expand' )(se)
 x = layers.multiply([x, se], name=name + 'se_excite')
 x = layers.Conv2D(
 filters_out,
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'project_conv' )(x)
 x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn' )(x)
 if id_skip and strides == 1 and filters_in == filters_out:
if drop_rate > 0:
x = layers.Dropout(
 drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop' )(x)
 x = layers.add([x, inputs], name=name + 'add')
 return x
@keras_export('keras.applications.efficientnet.EfficientNetB0',
'keras.applications.EfficientNetB0')
 def EfficientNetB0(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.0,
1.0,
224,
0.2,
model_name='efficientnetb0',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB1',
'keras.applications.EfficientNetB1')
 def EfficientNetB1(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.0,
1.1,
240,
0.2,
model_name='efficientnetb1',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB2',
'keras.applications.EfficientNetB2')
 def EfficientNetB2(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.1,
1.2,
260,
0.3,
model_name='efficientnetb2',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB3',
'keras.applications.EfficientNetB3')
 def EfficientNetB3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.2,
1.4,
300,
0.3,
model_name='efficientnetb3',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB4',
'keras.applications.EfficientNetB4')
 def EfficientNetB4(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.4,
1.8,
380,
0.4,
model_name='efficientnetb4',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB5',
'keras.applications.EfficientNetB5')
 def EfficientNetB5(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.6,
2.2,
456,
0.4,
model_name='efficientnetb5',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB6',
'keras.applications.EfficientNetB6')
 def EfficientNetB6(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 1.8,
2.6,
528,
0.5,
model_name='efficientnetb6',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.EfficientNetB7',
'keras.applications.EfficientNetB7')
 def EfficientNetB7(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
return EfficientNet(
 2.0,
3.1,
600,
0.5,
model_name='efficientnetb7',
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
input_shape=input_shape,
pooling=pooling,
classes=classes,
**kwargs)
 @keras_export('keras.applications.efficientnet.preprocess_input')
 def preprocess_input(x, data_format=None):
return x
@keras_export('keras.applications.efficientnet.decode_predictions')
 def decode_predictions(preds, top=5):

 return imagenet_utils.decode_predictions(preds, top=top )<choose_model_class>
test.isnull().sum().sum()
def get_model(weights='imagenet'):
inp = tf.keras.layers.Input(shape=(DIM,DIM,3))
base = EfficientNetB0(input_shape=(DIM,DIM,3),weights=None,include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(5,activation='softmax' )(x)
 model = tf.keras.Model(inputs=inp, outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.CategoricalCrossentropy(
 from_logits=False, label_smoothing=0.0001,
name='categorical_crossentropy' 
)
 model.compile(optimizer=opt,loss=loss,metrics=['categorical_accuracy'])
 return model<split>
test["MSZoning"].fillna("RL", inplace=True)
 train["LotFrontage"].fillna(0, inplace=True)
 test["LotFrontage"].fillna(0, inplace=True)
 train["Alley"].fillna("No_alley", inplace=True)
 test["Alley"].fillna("No_alley", inplace=True)
 train.drop("Utilities", axis=1, inplace=True)
 test.drop("Utilities", axis=1, inplace=True)
 test["Exterior1st"].fillna("VinylSd", inplace=True)
 test["Exterior2nd"].fillna("VinylSd", inplace=True)
 train["MasVnrType"].fillna("None", inplace=True)
 test["MasVnrType"].fillna("None", inplace=True)
 train["MasVnrArea"].fillna(0, inplace=True)
 test["MasVnrArea"].fillna(0, inplace=True)
 train["BsmtQual"].fillna("None", inplace=True)
 test["BsmtQual"].fillna("None", inplace=True)
 train["BsmtCond"].fillna("None", inplace=True)
 test["BsmtCond"].fillna("None", inplace=True)
 train["BsmtExposure"].fillna("None", inplace=True)
 test["BsmtExposure"].fillna("None", inplace=True)
 train["BsmtFinType1"].fillna("None", inplace=True)
 test["BsmtFinType1"].fillna("None", inplace=True)
 test["BsmtFinSF1"].fillna(0, inplace=True)
 train["BsmtFinType2"].fillna("None", inplace=True)
 test["BsmtFinType2"].fillna("None", inplace=True)
 test["BsmtFinSF2"].fillna(0, inplace=True)
 test["BsmtUnfSF"].fillna(0, inplace=True)
 test["TotalBsmtSF"].fillna(0, inplace=True)
 train["Electrical"].fillna("SBrkr", inplace=True)
 test["BsmtFullBath"].fillna(0, inplace=True)
 test["BsmtHalfBath"].fillna(0, inplace=True)
 test["KitchenQual"].fillna("TA", inplace=True)
 test["Functional"].fillna("Typ", inplace=True)
 train["FireplaceQu"].fillna("None", inplace=True)
 test["FireplaceQu"].fillna("None", inplace=True)
 train["GarageType"].fillna("None", inplace=True)
 test["GarageType"].fillna("None", inplace=True)
 train["GarageYrBlt"].fillna(0, inplace=True)
 test["GarageYrBlt"].fillna(0, inplace=True)
 train["GarageFinish"].fillna("None", inplace=True)
 test["GarageFinish"].fillna("None", inplace=True)
 test["GarageCars"].fillna(0, inplace=True)
 test["GarageArea"].fillna(0, inplace=True)
 train["GarageQual"].fillna("None", inplace=True)
 test["GarageQual"].fillna("None", inplace=True)
 train["GarageCond"].fillna("None", inplace=True)
 test["GarageCond"].fillna("None", inplace=True)
 train.drop("PoolQC", axis=1, inplace=True)
 test.drop("PoolQC", axis=1, inplace=True)
 train.drop("Fence", axis=1, inplace=True)
 test.drop("Fence", axis=1, inplace=True)
 train.drop("MiscFeature", axis=1, inplace=True)
 test.drop("MiscFeature", axis=1, inplace=True)
 test["SaleType"].fillna("WD", inplace=True )
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=12)
 for fold,(idxT,idxV)in enumerate(skf.split(np.arange(5))):
if fold==(FOLDS-1):
idxTT = idxT; idxVV = idxV
print('
print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV )<define_variables>
pred = np.zeros(( test_df.shape[0],5))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(5))):
print() ; print('
print('
print('
files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])
 files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])
 NUM_TRAINING_IMAGES = int(count_data_items(files_train))
NUM_VALIDATION_IMAGES = int(count_data_items(files_valid))
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))
model = get_model()
print('Loading best model...')
 model.load_weights('.. /input/cassavaefnetb0/fold-%i.h5'%fold)
 print('Predicting Test with TTA...')
 tta_steps = TTA
predictions = []
for i in tqdm(range(tta_steps)) :
preds = model.predict_generator(test_generator, verbose=1)
 predictions.append(preds)
 pred += np.mean(predictions, axis=0)/FOLDS
del model; z = gc.collect()<create_dataframe>
train.drop("Condition2", axis=1, inplace=True)
 test.drop("Condition2", axis=1, inplace=True)
 train.drop("RoofMatl", axis=1, inplace=True)
 test.drop("RoofMatl", axis=1, inplace=True)
 train.drop("LowQualFinSF", axis=1, inplace=True)
 test.drop("LowQualFinSF", axis=1, inplace=True)
 def remode_flg(x):
if x["YearBuilt"]==x["YearRemodAdd"]:
res = 1
else:
res = 0
return res
train["Remod_flg"] = train.apply(remode_flg, axis=1)
 test["Remod_flg"] = test.apply(remode_flg, axis=1)
 train["YearRemodAdd"] = train["YrSold"] - train["YearRemodAdd"]
test["YearRemodAdd"] = test["YrSold"] - test["YearRemodAdd"]
train["GarageYrBlt"] = train["YrSold"] - train["GarageYrBlt"]
test["GarageYrBlt"] = test["YrSold"] - test["GarageYrBlt"]
train.drop(["YearBuilt", "YrSold"], axis=1, inplace=True)
 test.drop(["YearBuilt", "YrSold"], axis=1, inplace=True)
 train.drop("TotalBsmtSF", axis=1, inplace=True)
 test.drop("TotalBsmtSF", axis=1, inplace=True)
 train["FullBathrooms"] = train["BsmtFullBath"] + train["FullBath"]
train.drop(["BsmtFullBath", "FullBath"], axis=1, inplace=True)
 test["FullBathrooms"] = test["BsmtFullBath"] + test["FullBath"]
test.drop(["BsmtFullBath", "FullBath"], axis=1, inplace=True)
 train["HalfBathrooms"] = train["BsmtHalfBath"] + train["HalfBath"]
train.drop(["BsmtHalfBath", "HalfBath"], axis=1, inplace=True)
 test["HalfBathrooms"] = test["BsmtHalfBath"] + test["HalfBath"]
test.drop(["BsmtHalfBath", "HalfBath"], axis=1, inplace=True)
 train["Rooms"] = train["TotRmsAbvGrd"] - train["BedroomAbvGr"]
train.drop("TotRmsAbvGrd", axis=1, inplace=True)
 test["Rooms"] = test["TotRmsAbvGrd"] - test["BedroomAbvGr"]
test.drop("TotRmsAbvGrd", axis=1, inplace=True)
 train["Porch"] = train['WoodDeckSF'] + train['OpenPorchSF'] + train["EnclosedPorch"] + train["3SsnPorch"] + train["ScreenPorch"]
test["Porch"] = test['WoodDeckSF'] + test['OpenPorchSF'] + test["EnclosedPorch"] + test["3SsnPorch"] + test["ScreenPorch"]
train.drop(["WoodDeckSF", "OpenPorchSF", "EnclosedPorch", "3SsnPorch", "ScreenPorch"], axis=1, inplace=True)
 test.drop(["WoodDeckSF", "OpenPorchSF", "EnclosedPorch", "3SsnPorch", "ScreenPorch"], axis=1, inplace=True)
 train["PoolArea"] = train["PoolArea"].apply(lambda x:1 if x>0 else 0)
 test["PoolArea"] = test["PoolArea"].apply(lambda x:1 if x>0 else 0)
 train['Fireplaces'] = train['Fireplaces'].apply(lambda x:1 if x>0 else 0)
 test['Fireplaces'] = test['Fireplaces'].apply(lambda x:1 if x>0 else 0)
 log_col = ["LotFrontage", "LotArea", "MasVnrArea", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF",
"1stFlrSF", "2ndFlrSF", "GrLivArea", "GarageArea", "Porch"]
for c in log_col:
train[c] = np.log(train[c]+1)
 test[c] = np.log(test[c]+1 )
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMAGE_SIZE[0],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
col_df = pd.DataFrame({"col":train.dtypes.index, "dtype":train.dtypes.values})
 obj_col = col_df[col_df["dtype"]=="object"]["col"].values
for i in obj_col:
ce_oe = ce.OneHotEncoder(cols=i, handle_unknown="impute")
 train = ce_oe.fit_transform(train)
 test = ce_oe.transform(test )
prediction = np.argmax(pred, axis=1)
 test_df['label'] = prediction
test_df = test_df[["image_id","label"]]
test_df.to_csv('submission.csv',index=False)
 test_df<define_variables>
X_train = train
X_test = test
y_train = target
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
y_train = np.log10(y_train )
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
%%time
tree = DecisionTreeRegressor(random_state=10)
 opt_tree = BayesSearchCV(
 tree,
{"max_depth":(5,20),
"max_leaf_nodes":(30,100)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_tree = opt_tree.fit(X_train, y_train)
 print("Best score:{}".format(bs_tree.best_score_))
print("Best params:{}".format(bs_tree.best_params_))
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'tf_efficientnet_b4_ns',
'img_size': 512,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
%%time
tree = DecisionTreeRegressor(max_depth=9, max_leaf_nodes=67, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(tree, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
%%time
forest = RandomForestRegressor(random_state=10)
 opt_forest = BayesSearchCV(
 forest,
{"max_depth":(5,40),
"max_leaf_nodes":(30,100),
"n_estimators":(50,100),
"min_samples_split":(0.00001,1.0),
"min_samples_leaf":(1,10)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_forest = opt_forest.fit(X_train, y_train)
 print("Best score:{}".format(bs_forest.best_score_))
print("Best params:{}".format(bs_forest.best_params_))
train.label.value_counts()<load_from_csv>
%%time
forest = RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(forest, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<categorify>
forest.fit(X_train, y_train)
 importance = forest.feature_importances_
indices = np.argsort(importance)[::-1]
for f in range(X_train.shape[1]):
print("%2d)%-*s %f" %(f+1, 30, X_train.columns[indices[f]], importance[indices[f]]))
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
%%time
scores = []
for i in range(50,len(indices),5):
forest = RandomForestRegressor()
sel_features = X_train.columns[indices[:i]]
X_tra, X_val, y_tra, y_val = train_test_split(X_train[sel_features], y_train, test_size=0.2, random_state=0)
 forest.fit(X_tra, y_tra)
 y_pred = forest.predict(X_val)
 score = r2_score(y_true=y_val, y_pred=y_pred)
 scores.append(score)
 plt.figure(figsize=(20,6))
plt.plot(range(50,len(indices),5), scores)
 plt.xlabel("Number of feature")
 plt.ylabel("Accuracy score" )
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
sel_features_forest = X_train.columns[indices[:130]]
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<split>
%%time
forest = RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(forest, X_train[sel_features_forest], y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/pytorch-efficientnet-baseline-train-amp-aug/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
del model
torch.cuda.empty_cache()<feature_engineering>
%%time
xgbr = xgb.XGBRegressor(random_state=10)
 opt_xgb = BayesSearchCV(
 xgbr,
{"learning_rate":(0.001,0.99),
"max_depth":(1,30),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_xgb = opt_xgb.fit(X_train, y_train)
 print("Best score:{}".format(bs_xgb.best_score_))
print("Best params:{}".format(bs_xgb.best_params_))
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
%%time
xgbr = xgb.XGBRegressor(learning_rate=0.1350, max_depth=3, subsample=0.7356, colsample_bytree=0.7682, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(xgbr, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
test.to_csv('submission.csv', index=False )<install_modules>
%%time
lgbm = lgb.LGBMRegressor()
opt_lgbm = BayesSearchCV(
 lgbm,
{"num_leaves":(20,100),
"n_estimators":(10,100),
"learning_rate":(0.001,1),
"max_depth":(1,30),
"min_split_gain":(0,0.5),
"min_child_weight":(0.001,0.1),
"min_child_samples":(5,50),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_lgbm = opt_lgbm.fit(X_train, y_train)
 print("Best score:{}".format(bs_lgbm.best_score_))
print("Best params:{}".format(bs_lgbm.best_params_))
!pip install.. /input/pytorch-image-models/timm-0.3.1-py3-none-any.whl<import_modules>
%%time
lgbm = lgb.LGBMRegressor(learning_rate=0.2660, max_depth=26, colsample_bytree=0.6411, min_child_samples=36, min_child_weight=0.09835, min_split_gain=0, n_estimators=100, num_leaves=25, subsample=0.1146)
 print("Cross validation score")
 cr_score = cross_val_score(lgbm, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
import numpy as np
import os
import pandas as pd
from fastai.vision.all import *
import albumentations<define_variables>
lgbm.fit(X_train, y_train)
 importance = lgbm.feature_importances_
indices = np.argsort(importance)[::-1]
for f in range(X_train.shape[1]):
print("%2d)%-*s %f" %(f+1, 30, X_train.columns[indices[f]], importance[indices[f]]))
set_seed(999,reproducible=True )<load_from_csv>
%%time
scores = []
for i in range(50,len(indices),5):
lgbm = lgb.LGBMRegressor()
sel_features = X_train.columns[indices[:i]]
X_tra, X_val, y_tra, y_val = train_test_split(X_train[sel_features], y_train, test_size=0.2, random_state=0)
 lgbm.fit(X_tra, y_tra)
 y_pred = lgbm.predict(X_val)
 score = r2_score(y_true=y_val, y_pred=y_pred)
 scores.append(score)
 plt.figure(figsize=(20,6))
plt.plot(range(50,len(indices),5), scores)
 plt.xlabel("Number of feature")
 plt.ylabel("Accuracy score" )
train_df = pd.read_csv(dataset_path/'train.csv' )<drop_column>
sel_features_lgbm = X_train.columns[indices[:100]]
train_df['path'] = train_df['image_id'].map(lambda x:dataset_path/'train_images'/x)
 train_df = train_df.drop(columns=['image_id'])
 train_df = train_df.sample(frac=1 ).reset_index(drop=True)
 train_df.head(10 )<train_model>
%%time
lgbm = lgb.LGBMRegressor(learning_rate=0.2660, max_depth=26, colsample_bytree=0.6411, min_child_samples=36, min_child_weight=0.09835, min_split_gain=0, n_estimators=100, num_leaves=25, subsample=0.1146)
 print("Cross validation score")
 cr_score = cross_val_score(lgbm, X_train[sel_features_lgbm], y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
im = Image.open(train_df['path'][1])
 width, height = im.size
print(width,height )<categorify>
sc = StandardScaler()
sc.fit(X_train)
 X_train_std = sc.transform(X_train)
 X_test_std = sc.transform(X_test )
class AlbumentationsTransform(RandTransform):
"A transform handler for multiple `Albumentation` transforms"
split_idx,order=None,2
def __init__(self, train_aug, valid_aug): store_attr()
def before_call(self, b, split_idx):
self.idx = split_idx
def encodes(self, img: PILImage):
if self.idx == 0:
aug_img = self.train_aug(image=np.array(img)) ['image']
else:
aug_img = self.valid_aug(image=np.array(img)) ['image']
return PILImage.create(aug_img )<normalization>
ridge = Ridge()
opt_r = BayesSearchCV(
 ridge,
{"alpha":(0.0001,1000)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_r = opt_r.fit(X_train_std, y_train)
 print("Best score:{}".format(bs_r.best_score_))
print("Best params:{}".format(bs_r.best_params_))
def get_train_aug(sz): return albumentations.Compose([
albumentations.RandomResizedCrop(sz,sz),
albumentations.Transpose(p=0.5),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.ShiftScaleRotate(p=0.5),
albumentations.HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
albumentations.RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
albumentations.CoarseDropout(p=0.5),
albumentations.Cutout(p=0.5)
 ])
 def get_valid_aug(sz): return albumentations.Compose([
albumentations.CenterCrop(sz,sz, p=1.) ,
albumentations.Resize(sz,sz)
 ], p=1.)<create_dataframe>
ridge = Ridge(alpha=270.2)
 print("Cross validation score")
 cr_score = cross_val_score(ridge, X_train_std, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
def get_dls(sz,bs):
item_tfms = AlbumentationsTransform(get_train_aug(sz), get_valid_aug(sz))
batch_tfms = [Normalize.from_stats(*imagenet_stats)]
dls = ImageDataLoaders.from_df(train_df,
valid_pct=0.2,
seed=999,
label_col=0,
fn_col=1,
bs=bs,
item_tfms=item_tfms,
batch_tfms=batch_tfms)
 return dls<set_options>
lasso = Lasso()
opt_l = BayesSearchCV(
 lasso,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_l = opt_l.fit(X_train_std, y_train)
 print("Best score:{}".format(bs_l.best_score_))
print("Best params:{}".format(bs_l.best_params_))
dls = get_dls(456,16 )<define_variables>
lasso = Lasso(alpha=0.00001)
 print("Cross validation score")
 cr_score = cross_val_score(lasso, X_train_std, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
dls.show_batch()<categorify>
elas = ElasticNet()
opt_e = BayesSearchCV(
 elas,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_e = opt_e.fit(X_train_std, y_train)
 print("Best score:{}".format(bs_e.best_score_))
print("Best params:{}".format(bs_e.best_params_))
def create_timm_body(arch:str, pretrained=True, cut=None, n_in=3):
"Creates a body from any model in the `timm` library."
model = create_model(arch, pretrained=pretrained, num_classes=0, global_pool='')
 _update_first_layer(model, n_in, pretrained)
 if cut is None:
ll = list(enumerate(model.children()))
cut = next(i for i,o in reversed(ll)if has_pool_type(o))
if isinstance(cut, int): return nn.Sequential(*list(model.children())[:cut])
 elif callable(cut): return cut(model)
 else: raise NamedError("cut must be either integer or function")
 def create_timm_model(arch:str, n_out, cut=None, pretrained=True, n_in=3, init=nn.init.kaiming_normal_, custom_head=None,
concat_pool=True, **kwargs):
"Create custom architecture using `arch`, `n_in` and `n_out` from the `timm` library"
body = create_timm_body(arch, pretrained, None, n_in)
 if custom_head is None:
nf = num_features_model(nn.Sequential(*body.children())) *(2 if concat_pool else 1)
 head = create_head(nf, n_out, concat_pool=concat_pool, **kwargs)
 else: head = custom_head
model = nn.Sequential(body, head)
 if init is not None: apply_init(model[1], init)
 return model<choose_model_class>
elas = ElasticNet(alpha=0.00365)
 print("Cross validation score")
 cr_score = cross_val_score(elas, X_train_std, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
def timm_learner(dls, arch:str, loss_func=None, pretrained=True, cut=None, splitter=None,
y_range=None, config=None, n_out=None, normalize=True, **kwargs):
"Build a convnet style learner from `dls` and `arch` using the `timm` library"
if config is None: config = {}
if n_out is None: n_out = get_c(dls)
 assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
 model = create_timm_model(arch, n_out, default_split, pretrained, y_range=y_range, **config)
 learn = Learner(dls, model, loss_func=loss_func, splitter=default_split, **kwargs)
 if pretrained: learn.freeze()
return learn<choose_model_class>
%%time
estimators = [("tr", DecisionTreeRegressor(max_depth=9, max_leaf_nodes=67, random_state=10)) , 
("rf", RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)) , 
("xg", xgb.XGBRegressor(learning_rate=0.1350, max_depth=3, subsample=0.7356, colsample_bytree=0.7682, random_state=10)) , 
("lg", lgb.LGBMRegressor(learning_rate=0.2660, max_depth=26, colsample_bytree=0.6411, min_child_samples=36, min_child_weight=0.09835, min_split_gain=0, n_estimators=100, num_leaves=25, subsample=0.1146)) , 
("ri", Ridge(alpha=5.91)) , 
("la", Lasso(alpha=0.00001)) , 
("el", ElasticNet(alpha=0.00069)) ]
sreg = StackingRegressor(estimators=estimators)
 sreg.fit(X_train, y_train)
 print("Score:{}".format(sreg.score(X_train, y_train)))
 print("Params:{}".format(sreg.get_params))
learn = timm_learner(dls,
'tf_efficientnet_b5_ns',
opt_func=ranger,
loss_func=LabelSmoothingCrossEntropy() ,
cbs=[GradientAccumulation(n_acc=32)],
metrics = [accuracy] ).to_native_fp16()<find_best_params>
%%time
print("Cross validation score")
 cr_score = cross_val_score(sreg, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
learn.lr_find()<train_model>
ps_test = sreg.predict(X_test)
 X_test["y:pseudo"] = ps_test
learn.freeze()
learn.fit_flat_cos(1,1e-1, wd=0.1, cbs=[MixUp() ] )<save_model>
Ps_X_train = X_test.sample(int(len(X_train)/2))
Ps_X_train.reset_index(drop=True, inplace=True)
 Ps_y_train = Ps_X_train["y:pseudo"]
Ps_X_train = Ps_X_train.drop("y:pseudo", axis=1)
 n_X_train = pd.concat([X_train, Ps_X_train] ).reset_index(drop=True)
 n_y_train = pd.concat([y_train, Ps_y_train] ).reset_index(drop=True )
learn.save('stage-1' )<load_pretrained>
%%time
tree = DecisionTreeRegressor(random_state=10)
 opt_tree = BayesSearchCV(
 tree,
{"max_depth":(5,20),
"max_leaf_nodes":(30,100)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_tree = opt_tree.fit(n_X_train, n_y_train)
 print("Best score:{}".format(bs_tree.best_score_))
print("Best params:{}".format(bs_tree.best_params_))
learn = learn.load('stage-1' )<find_best_params>
%%time
tree = DecisionTreeRegressor(max_depth=10, max_leaf_nodes=72, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(tree, n_X_train, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
learn.unfreeze()
learn.lr_find()<train_model>
%%time
forest = RandomForestRegressor(random_state=10)
 opt_forest = BayesSearchCV(
 forest,
{"max_depth":(5,40),
"max_leaf_nodes":(30,100),
"n_estimators":(50,100),
"min_samples_split":(0.00001,1.0),
"min_samples_leaf":(1,10)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_forest = opt_forest.fit(n_X_train[sel_features_forest], n_y_train)
 print("Best score:{}".format(bs_forest.best_score_))
print("Best params:{}".format(bs_forest.best_params_))
learn.unfreeze()
learn.fit_flat_cos(10, 1e-3,cbs=[MixUp() ,SaveModelCallback() ] )<categorify>
%%time
forest = RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(forest, n_X_train[sel_features_forest], n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
learn = learn.to_native_fp32()<save_model>
%%time
xgbr = xgb.XGBRegressor(random_state=10)
 opt_xgb = BayesSearchCV(
 xgbr,
{"learning_rate":(0.001,0.99),
"max_depth":(1,30),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_xgb = opt_xgb.fit(n_X_train, n_y_train)
 print("Best score:{}".format(bs_xgb.best_score_))
print("Best params:{}".format(bs_xgb.best_params_))
learn.save('stage-2' )<load_from_csv>
%%time
xgbr = xgb.XGBRegressor(learning_rate=0.07403, max_depth=211, subsample=0.7613, colsample_bytree=0.7985, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(xgbr, n_X_train, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
sample_df = pd.read_csv(dataset_path/'sample_submission.csv')
 sample_df.head()<categorify>
%%time
lgbm = lgb.LGBMRegressor()
opt_lgbm = BayesSearchCV(
 lgbm,
{"num_leaves":(20,100),
"n_estimators":(10,100),
"learning_rate":(0.001,1),
"max_depth":(1,30),
"min_split_gain":(0,0.5),
"min_child_weight":(0.001,0.1),
"min_child_samples":(5,50),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_lgbm = opt_lgbm.fit(n_X_train[sel_features_lgbm], n_y_train)
 print("Best score:{}".format(bs_lgbm.best_score_))
print("Best params:{}".format(bs_lgbm.best_params_))
_sample_df = sample_df.copy()
_sample_df['path'] = _sample_df['image_id'].map(lambda x:dataset_path/'test_images'/x)
 _sample_df = _sample_df.drop(columns=['image_id'])
 test_dl = dls.test_dl(_sample_df )<define_variables>
%%time
lgbm = lgb.LGBMRegressor(learning_rate=0.1297, max_depth=20, colsample_bytree=0.9295, min_child_samples=43, min_child_weight=0.0484, min_split_gain=0, n_estimators=77, num_leaves=22, subsample=0.1)
 print("Cross validation score")
 cr_score = cross_val_score(lgbm, n_X_train[sel_features_lgbm], n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
test_dl.show_batch()<normalization>
n_X_train_std = sc.transform(n_X_train )
preds, _ = learn.tta(dl=test_dl, n=15, beta=0 )<feature_engineering>
%%time
ridge = Ridge()
opt_r = BayesSearchCV(
 ridge,
{"alpha":(0.0001,1000)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_r = opt_r.fit(n_X_train_std, n_y_train)
 print("Best score:{}".format(bs_r.best_score_))
print("Best params:{}".format(bs_r.best_params_))
sample_df['label'] = preds.argmax(dim=-1 ).numpy()<save_to_csv>
%%time
ridge = Ridge(alpha=70.98)
 print("Cross validation score")
 cr_score = cross_val_score(ridge, n_X_train_std, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
sample_df.to_csv('submission.csv',index=False )<set_options>
%%time
lasso = Lasso()
opt_l = BayesSearchCV(
 lasso,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_l = opt_l.fit(n_X_train_std, n_y_train)
 print("Best score:{}".format(bs_l.best_score_))
print("Best params:{}".format(bs_l.best_params_))
%matplotlib inline 
<load_from_csv>
%%time
lasso = Lasso(alpha=0.00012)
 print("Cross validation score")
 cr_score = cross_val_score(lasso, n_X_train_std, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
train = pd.read_csv('.. /input/train.csv')
 print('Shape of train set {}'.format(train.shape))
test = pd.read_csv('.. /input/test.csv')
 print('Shape of test set {}'.format(test.shape))<categorify>
%%time
elas = ElasticNet()
opt_e = BayesSearchCV(
 elas,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_e = opt_e.fit(n_X_train_std, n_y_train)
 print("Best score:{}".format(bs_e.best_score_))
print("Best params:{}".format(bs_e.best_params_))
def onehot_encode(cat_feat, data):

 encoded = pd.get_dummies(data[cat_feat], prefix=cat_feat, dummy_na=True)
 res = pd.concat([data.drop(columns=[cat_feat]), encoded], axis='columns')
 return res
def add_quality(df, componente='pared', component='wall'):
for i in [1,2,3]:
i_quality =(df['e{}{}'.format(componente, i)] == 1)
 df.loc[i_quality, '{}_quality'.format(component)] = i
return df
def to_english(df, sp_pre='pared', eng_pre='wall_', translate=None):

 for sp in translate.keys() :
spanish_name = sp_pre + '{}'.format(sp)
 english_name = eng_pre + '{}'.format(translate[sp])
 df.rename(columns={spanish_name: english_name}, inplace=True)
 return df<define_variables>
%%time
elas = ElasticNet(alpha=0.00011)
 print("Cross validation score")
 cr_score = cross_val_score(elas, n_X_train_std, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
def mk_derived_feats(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 return df
def mk_agg_feats(df):
aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['sum', 'mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'sum']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
return df
def drop_redundant(df):
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['hhsize', 'area2'], axis=1, inplace=True)
 return df<count_unique_values>
%%time
estimators = [("tr", DecisionTreeRegressor(max_depth=10, max_leaf_nodes=72, random_state=10)) , 
("rf", RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)) , 
("xg", xgb.XGBRegressor(learning_rate=0.07403, max_depth=211, subsample=0.7613, colsample_bytree=0.7985, random_state=10)) , 
("lg", lgb.LGBMRegressor(learning_rate=0.1297, max_depth=20, colsample_bytree=0.9295, min_child_samples=43, min_child_weight=0.0484, min_split_gain=0, n_estimators=77, num_leaves=22, subsample=0.1)) , 
("ri", Ridge(alpha=0.0001)) , 
("la", Lasso(alpha=0.00001)) , 
("el", ElasticNet(alpha=0.00001)) ]
sreg = StackingRegressor(estimators=estimators)
 sreg.fit(n_X_train, n_y_train)
 print("Score:{}".format(sreg.score(n_X_train, n_y_train)))
 print("Params:{}".format(sreg.get_params))
test['Target'] = np.nan
data_all = pd.concat([train, test])
 n_house = data_all['idhogar'].nunique()
print('<categorify>
print("Cross validation score")
 cr_score = cross_val_score(sreg, n_X_train, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))
yes_no_map = {'no': 0, 'yes': 1}
data_all['dependency'] = data_all['dependency'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefe'] = data_all['edjefe'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefa'] = data_all['edjefa'].replace(yes_no_map ).astype(np.float32)
 data_all['idhogar'] = LabelEncoder().fit_transform(data_all['idhogar'])
 data_all = mk_derived_feats(data_all)
 data_all = mk_agg_feats(data_all)
 data_all = drop_redundant(data_all )<define_variables>
X_test.drop("y:pseudo", axis=1, inplace=True)
 X_test_std = sc.transform(X_test )
fe_feats = [ff for ff in data_all.columns if ff.startswith('fe_')]
agg_feats = [ff for ff in data_all.columns if ff.startswith('agg')]
basic_feats = ['dependency']<define_variables>
y_pred_tree = 10**(bs_tree.predict(X_test))
y_pred_forest = 10**(bs_forest.predict(X_test[sel_features_forest]))
y_pred_xgb = 10**(bs_xgb.predict(X_test))
y_pred_lgbm = 10**(bs_lgbm.predict(X_test[sel_features_lgbm]))
y_pred_r = 10**(bs_r.predict(X_test_std))
y_pred_l = 10**(bs_l.predict(X_test_std))
y_pred_sreg = 10**(sreg.predict(X_test))
too_high_edu_levels = [ff for ff in agg_feats if('instlevel9' in ff)or('instlevel8' in ff)]
irrelevant_members = [ff for ff in agg_feats if('parentesco9' in ff)or('parentesco5' in ff)
 or('parentesco11' in ff)or('parentesco6' in ff)]
to_drop = too_high_edu_levels + irrelevant_members
to_drop<drop_column>
submit_xgb = sample.copy()
submit_xgb["SalePrice"] = y_pred_xgb
for ff in to_drop:
agg_feats.remove(ff )<filter>
submit_lgbm = sample.copy()
submit_lgbm["SalePrice"] = y_pred_lgbm
is_head =(data_all.parentesco1 == 1)
 head_df = data_all.loc[is_head, :]<categorify>
submit_sreg = sample.copy()
submit_sreg["SalePrice"] = y_pred_sreg
head_df.loc[head_df['male'] == 1, 'head_gender'] = 'male'
head_df.loc[head_df['female'] == 1, 'head_gender'] = 'female'
head_df = onehot_encode('head_gender', head_df)
 head_gender_feats = [cc for cc in head_df.columns if 'head_gender' in cc]<feature_engineering>
submit_ems = sample.copy()
submit_ems["SalePrice"] = y_pred_tree*0.05 + y_pred_forest*0.1 + y_pred_xgb*0.3 + y_pred_lgbm*0.2 + y_pred_r*0.05 + y_pred_l*0.05 + y_pred_sreg*0.25
<merge><EOS> 
submit_xgb.to_csv('xgb_submission.csv', index=False)
 submit_lgbm.to_csv('lgbm_submission.csv', index=False)
 submit_sreg.to_csv('stack_submission.csv', index=False)
 submit_ems.to_csv('ems_submission.csv', index=False)
 print("Your submission was successfully saved!" )