Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<count_values>
optuna.logging.set_verbosity(optuna.logging.CRITICAL )
df_train = not_cactus[:3491]
df_train = df_train.append(has_cactus[:3491])
 df_train = df_train.sample(frac=1 ).reset_index(drop=True)
 df_train['has_cactus'].value_counts()<count_values>
data = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv', index_col=0)
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv', index_col=0)
 preds = data.columns[:-1]
target = data.columns[-1]
df_valid = not_cactus[3491:]
df_valid = df_valid.append(has_cactus[3491:3491 + 873*3])
 df_valid = df_valid.sample(frac=1 ).reset_index(drop=True)
 df_valid['has_cactus'].value_counts()<init_hyperparams>
cat_cols = [col for col in preds if 'cat' in col]
data[cat_cols] = data[cat_cols].astype('category')
 test[cat_cols] = test[cat_cols].astype('category' )
train_datagen = ImageDataGenerator(rotation_range=50,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
rescale=1./255)
 valid_datagen = ImageDataGenerator(rescale=1./255 )<create_dataframe>
def objective(trial):
n_estimators = trial.suggest_int('n_estimators', 100, 1000)
 scores = []
kf = KFold(5)
 for i,(train_idx, test_idx)in enumerate(kf.split(data)) :
X_train = data.iloc[train_idx][preds]
y_train = data.iloc[train_idx][target]
X_test = data.iloc[test_idx][preds]
y_test = data.iloc[test_idx][target]
estimator = LGBMRegressor(n_estimators=n_estimators)
 estimator.fit(X_train,
y_train,
eval_set=(X_test, y_test),
eval_metric='rmse',
categorical_feature=cat_cols,
verbose=0)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 return np.mean(scores )
new_image_size = 64
batch_size = 128
train_generator = train_datagen.flow_from_dataframe(df_train,
directory=train_dir,
x_col='id',
y_col='has_cactus',
target_size=(new_image_size, new_image_size),
class_mode='binary',
batch_size=batch_size)
 valid_generator = valid_datagen.flow_from_dataframe(df_valid,
directory=train_dir,
x_col='id',
y_col='has_cactus',
target_size=(new_image_size, new_image_size),
class_mode='binary',
batch_size=batch_size )<choose_model_class>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, timeout=3600*0.5 )
model = Sequential()
model.add(Conv2D(32,(3, 3), input_shape=(new_image_size, new_image_size, 3), activation='relu'))
model.add(Dropout(0.2))
model.add(MaxPooling2D(( 2,2)))
 model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(Dropout(0.2))
model.add(MaxPooling2D(( 2,2)))
 model.add(Conv2D(128,(3,3), activation='relu'))
model.add(Dropout(0.2))
model.add(MaxPooling2D(( 2,2)))
 model.add(Conv2D(512,(3,3), activation='relu'))
model.add(Dropout(0.2))
model.add(MaxPooling2D(( 2,2)))
 model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()<choose_model_class>
study.best_value
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'] )<choose_model_class>
best_params = study.best_params
best_params
checkpoint = ModelCheckpoint("model.hdf5",
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
 reduce_lr = ReduceLROnPlateau(monitor='val_acc',
factor=.2,
patience=10,
min_lr=1e-9,
verbose=1)
 early_stop = EarlyStopping(monitor='val_acc',
patience=100,
verbose=1,
mode='auto',
baseline=None,
restore_best_weights=True )<compute_test_metric>
def objective(trial):
hyper_params = {
'num_leaves': trial.suggest_int('num_leaves', 1, 63),
'min_data_in_leaf': trial.suggest_int('min_data_in_leaf ', 1, 100)
 }
scores = []
kf = KFold(5)
 for i,(train_idx, test_idx)in enumerate(kf.split(data)) :
X_train = data.iloc[train_idx][preds]
y_train = data.iloc[train_idx][target]
X_test = data.iloc[test_idx][preds]
y_test = data.iloc[test_idx][target]
hyper_params.update(best_params)
 estimator = LGBMRegressor(**hyper_params)
 estimator.fit(X_train,
y_train,
eval_set=(X_test, y_test),
eval_metric='rmse',
verbose=0)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 return np.mean(scores )
model.evaluate_generator(valid_generator, steps=len(df_valid)//batch_size )<train_model>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, timeout=3600*2.5 )
test_fnames = []
test_imgs = []
for fname in os.listdir(test_dir):
test_fnames.append(fname)
 img = load_img(test_dir + fname, target_size=(new_image_size, new_image_size, 3))
img = img_to_array(img)/ 255
test_imgs.append(img)
 test_imgs = np.array(test_imgs)
 test_imgs.shape<predict_on_test>
study.best_value
pred = model.predict(test_imgs)
 pred[:5]<save_to_csv>
best_params.update(study.best_params)
 best_params
submission = pd.DataFrame()
submission['id'] = test_fnames
submission['has_cactus'] = pred
submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
def objective(trial):
hyper_params = {
'bagging_freq': trial.suggest_int('bagging_freq', 1, 100),
'bagging_fraction': trial.suggest_float('bagging_fraction ', 0, 1.0),
'feature_fraction': trial.suggest_float('feature_fraction', 0, 1.0)
 }
scores = []
kf = KFold(5)
 for i,(train_idx, test_idx)in enumerate(kf.split(data)) :
X_train = data.iloc[train_idx][preds]
y_train = data.iloc[train_idx][target]
X_test = data.iloc[test_idx][preds]
y_test = data.iloc[test_idx][target]
hyper_params.update(best_params)
 estimator = LGBMRegressor(**hyper_params)
 estimator.fit(X_train,
y_train,
eval_set=(X_test, y_test),
eval_metric='rmse',
verbose=0)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 return np.mean(scores )
%matplotlib inline
style.use('fivethirtyeight')
 sns.set(style='whitegrid', color_codes=True)
 print(os.listdir(".. /input")) 
<define_variables>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, timeout=3600*2 )
random.seed(42)
 np.random.seed(42 )<load_from_csv>
study.best_value
PATH = ".. /input"
IMAGE_SIZE =(32, 32)
 train_dir=f'{PATH}/train/train'
test_dir=f'{PATH}/test/test'
train_img = os.listdir(train_dir)
 test_img = os.listdir(test_dir)
 train_df =pd.read_csv(f'{PATH}/train.csv')
 test_df = pd.read_csv(f'{PATH}/sample_submission.csv')
 train_new_dir = 'train'
if os.path.exists(train_new_dir):
shutil.rmtree(train_new_dir)
 else:
os.makedirs(train_new_dir)
 print(f"The number of rows in train and test set are {len(train_df)} and {len(test_df)}" )<define_variables>
best_params.update(study.best_params)
 best_params
print(f"The number of rows in train and test set are {len(os.listdir(train_new_dir)) } and {len(test_df)}" )<split>
def objective(trial):
hyper_params = {
'lambda_l1': trial.suggest_float('lambda_l1', 1E-12, 20, log=True),
'lambda_l2': trial.suggest_float('lambda_l2', 1E-12, 20, log=True)
 }
scores = []
kf = KFold(5)
 for i,(train_idx, test_idx)in enumerate(kf.split(data)) :
X_train = data.iloc[train_idx][preds]
y_train = data.iloc[train_idx][target]
X_test = data.iloc[test_idx][preds]
y_test = data.iloc[test_idx][target]
hyper_params.update(best_params)
 estimator = LGBMRegressor(**hyper_params)
 estimator.fit(X_train,
y_train,
eval_set=(X_test, y_test),
eval_metric='rmse',
verbose=0)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 return np.mean(scores )
train, valid = train_test_split(train_df, stratify=train_df.has_cactus, test_size=0.33, random_state=2019 )<choose_model_class>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, timeout=3600*2 )
drop_rate=0.2
model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(32,32,3)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Dropout(drop_rate))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Dropout(drop_rate))
model.add(GlobalAveragePooling2D())
 model.add(BatchNormalization())
 model.add(Dropout(drop_rate))
model.add(Dense(1,activation='sigmoid'))<choose_model_class>
study.best_value
model.compile(loss='binary_crossentropy',optimizer=Adam() , metrics=['accuracy'] )<choose_model_class>
best_params.update(study.best_params)
 best_params
file_path = 'best_weights.h5'
callbacks = [
ModelCheckpoint(file_path, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max'),
ReduceLROnPlateau(monitor = 'val_loss', factor = 0.8, patience = 4, verbose = 1, mode = 'min', min_lr = 1e-8),
EarlyStopping(monitor = 'val_loss', min_delta = 1e-10, patience = 32, verbose = 1, restore_best_weights = True)
 ]<train_model>
k = 10
test[target] = 0
scores = []
kf = KFold(k)
 for i,(train_idx, test_idx)in enumerate(kf.split(data)) :
X_train = data.iloc[train_idx][preds]
y_train = data.iloc[train_idx][target]
X_test = data.iloc[test_idx][preds]
y_test = data.iloc[test_idx][target]
best_params['learning_rate'] = 0.005
best_params['n_estimators'] = 100000
estimator = LGBMRegressor(**best_params)
 estimator.fit(X_train,
y_train,
eval_set=(X_test, y_test),
eval_metric='rmse',
early_stopping_rounds=1000,
verbose=1000)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 test[target] += estimator.predict(test[preds])/ k
test[target].to_csv('submission.csv' )
epochs=128
history=model.fit_generator(train_generator,
steps_per_epoch=train_generator.n//train_generator.batch_size,
epochs=epochs,
verbose = 1,
shuffle=True,
validation_data=valid_generator,
validation_steps=valid_generator.n//valid_generator.batch_size,
callbacks = callbacks )<load_pretrained>
print(f"Expected score: {np.mean(scores)}" )