Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<filter>
import xgboost as xgb
import numpy as np
import pandas as pd
import random
import optuna
from sklearn.model_selection import KFold, train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
train = pd.read_csv(".. /input/tabular-playground-series-feb-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-feb-2021/test.csv" )
test_df.to_csv('submission.csv', index=False )<define_variables>
df=train
for c in df.columns:
if df[c].dtype=='object':
lbl = LabelEncoder()
df[c]=df[c].fillna('N')
 lbl.fit(list(df[c].values))
df[c] = lbl.transform(df[c].values)
 train=df
TRAIN_IMAGES_PATH = '.. /input/train/train'
TEST_IMAGES_PATH = '.. /input/test/test'<load_from_csv>
df=test
for c in df.columns:
if df[c].dtype=='object':
lbl = LabelEncoder()
df[c]=df[c].fillna('N')
 lbl.fit(list(df[c].values))
df[c] = lbl.transform(df[c].values)
 test=df
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/sample_submission.csv')
 train_image_ids = train_df['id']
training_labels = train_df['has_cactus']
test_image_ids = test_df['id']
test_labels = test_df['has_cactus']<categorify>
target = train['target']
data = train.drop(['target','id'],axis=1 )
all_train_images, normalized_images = get_images(TRAIN_IMAGES_PATH, train_image_ids)
 test_images, normalized_test_images = get_images(TEST_IMAGES_PATH, test_image_ids )<categorify>
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)
 param = {
'objective': trial.suggest_categorical('objective',['reg:tweedie']),
'tree_method': trial.suggest_categorical('tree_method',['hist']),
'lambda': trial.suggest_loguniform('lambda',1e-3,10.0),
'alpha': trial.suggest_loguniform('alpha',1e-3,10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.01,0.012,0.014,0.016,0.018,0.02]),
'n_estimators': trial.suggest_categorical('n_estimators', [1000,2000,4000,8000]),
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20]),
'random_state': trial.suggest_categorical('random_state', [24,48,2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1,300),
'use_label_encoder': trial.suggest_categorical('use_label_encoder',[False])
 }
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
augs = [np.fliplr, np.flipud, np.rot90]
def augment(images, labels, augs):

 all_images = list()
all_labels = list()
for i,image in tqdm_notebook(enumerate(images)) :
all_images.append(image)
 cur_label = labels[i]
all_labels.append(cur_label)
 if cur_label == 1:
all_images.append(augs[random.randint(0,2)](image))
all_labels.append(cur_label)
 else:
for aug in augs:
all_labels.append(cur_label)
 all_images.append(aug(image))
return np.stack(all_images), np.array(all_labels)
<categorify>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=16)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
normalized_train_images, final_training_labels = augment(normalized_images, np.array(training_labels), augs)
 NUM_TRAIN_IMAGES = int(0.75 * normalized_train_images.shape[0])
 indices = np.random.permutation(normalized_train_images.shape[0])
 training_idx, val_idx = indices[:NUM_TRAIN_IMAGES], indices[NUM_TRAIN_IMAGES:]
train_data = normalized_train_images[training_idx,:]
train_labels = np.array(final_training_labels)[training_idx]
val_data = normalized_train_images[val_idx,:]
val_labels = final_training_labels[val_idx]<categorify>
study.trials_dataframe()
def get_weights(shape):

 initializer = tf.contrib.layers.xavier_initializer()
return tf.Variable(initializer(shape=shape))
def get_biases(length):

 return tf.Variable(tf.constant(0.0005, shape=[length])) 
<categorify>
Best_trial=study.best_trial.params
print(Best_trial )
def conv_layer(input, in_channels, filter_size, num_filters):

 shape = [filter_size, filter_size, in_channels, num_filters]
weights = get_weights(shape)
 bias = get_biases(num_filters)
 layer = tf.nn.convolution(input,
weights,
strides=[1,1], dilation_rate=[1,1],
padding="SAME")
 layer= tf.nn.bias_add(layer, bias)
 new_layer = tf.nn.relu(layer)
 print(new_layer)
 return new_layer, weights
def flatten(input_tensor):

 layer_shape = input_tensor.get_shape()
total_elements = layer_shape[1:4].num_elements()
layer = tf.reshape(input_tensor, [-1, total_elements])
 return layer, total_elements
def fc_layer(input, in_features, out_features):

 weight = get_weights([in_features, out_features])
 bias = get_biases(out_features)
 layer = tf.matmul(input, weight)+ bias
return layer<init_hyperparams>
sample = pd.read_csv(".. /input/tabular-playground-series-feb-2021/sample_submission.csv")
 print(sample.shape )
input_image = tf.placeholder(name="input", shape=(None, 32, 32, 3), dtype=tf.float32)
 labels = tf.placeholder(name="labels", shape=(None), dtype=tf.int64)
 with tf.variable_scope("block1_conv1"):
layer_conv1, weights_1 = conv_layer(input_image, 3, 3, 32)
 with tf.variable_scope("block1_conv2"):
layer_conv2, weights_2 = conv_layer(layer_conv1, 32, 3, 32)
 with tf.variable_scope("block2_conv1"):
layer_conv3, weights_3 = conv_layer(layer_conv2, 32, 3, 64)
 with tf.variable_scope("block2_conv2"):
layer_conv4, weights_4 = conv_layer(layer_conv3, 64, 3, 64)
 layer_output_pool = tf.nn.max_pool(layer_conv4, ksize = [1,4,4,1], strides = [1,2,2,1], padding = "VALID")
 with tf.variable_scope("block3_conv1"):
layer_conv4, weights_4 = conv_layer(layer_output_pool, 64, 3, 128)
 with tf.variable_scope("block3_conv2"):
layer_conv5, weights_5 = conv_layer(layer_conv4, 128, 3, 128)
 with tf.variable_scope("block3_conv3"):
layer_conv6, weights_6 = conv_layer(layer_conv5, 128, 3, 128)
 layer_output_pool = tf.nn.max_pool(layer_conv6, ksize = [1,4,4,1], strides = [1,2,2,1], padding = "VALID")
 print(layer_output_pool, layer_conv4)
 flattened_layer, in_features = flatten(layer_output_pool)
 fclyr = fc_layer(flattened_layer, in_features, 128)
 fc_layer2 = tf.nn.relu(fclyr)
 final_layer = fc_layer(fc_layer2, 128, 2)
 y_pred = tf.nn.softmax(final_layer)
 cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=final_layer,
labels=tf.one_hot(labels,2))
cost = tf.reduce_mean(cross_entropy)
 optimizer = tf.train.AdamOptimizer(learning_rate=0.001 ).minimize(cost)
 gpu_options = tf.GPUOptions(allow_growth=True)
 config = tf.ConfigProto(gpu_options=gpu_options)
 init=tf.global_variables_initializer()<init_hyperparams>
preds = np.zeros(( sample.shape[0]))
kf = KFold(n_splits=5,random_state=48,shuffle=True)
 for trn_idx, test_idx in kf.split(train[columns],target):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=target.iloc[trn_idx],target.iloc[test_idx]
model = xgb.XGBRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse=mean_squared_error(y_val, model.predict(X_val),squared=False)
 print(rmse )
<define_variables><EOS> 
subm = sample
subm['target'] = preds
subm.to_csv('submission.csv',index=False)
 subm