Kaggle competition: Tabular Playground Series - Jan 2021
test_images = glob('.. /input/fish-data/test_stg1/test_stg1/*.jpg')[:4]
plt.figure(figsize=(6, 4 * len(test_images)) , dpi=240)
 for i, filename in enumerate(test_images):
_, sample_img = load_img(filename)
 pred = model.predict(np.array([sample_img,]))
decoded = decode_prediction(pred[0], conf_thr=0.1)
 decoded = non_max_suppression(decoded,
max_output_size=1,
iou_threshold=0.5)
 plt.subplot(len(test_images), 1, i + 1)
 img = draw_boxes({'filename': filename}, decoded,(IMG_WIDTH, IMG_HEIGHT))
plt.imshow(img)
 plt.title('Предсказание на тесте {}'.format(filename.split('/')[-1])) ;<categorify>
train = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv', index_col='id')
 test = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv', index_col='id' )
def make_predictions() :
ptable = pd.DataFrame(columns=['image', 'ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK','YFT'])
 for i, file in enumerate(tqdm(glob('.. /input/fish-data/test_stg1/test_stg1/*.jpg'))):
bn = os.path.basename(file)
 _, sample_img = load_img(file)
 pred = model.predict(np.array([sample_img,])) [0]
decoded = decode_prediction(pred, conf_thr=0.015)
 decoded = non_max_suppression(decoded,
max_output_size=1,
iou_threshold=0.5)
 decoded = decoded[0]['labels']
ptable.loc[i, 'image'] = bn
ptable.loc[i, 'ALB'] = decoded[5]
ptable.loc[i, 'BET'] = decoded[1]
ptable.loc[i, 'DOL'] = decoded[4]
ptable.loc[i, 'LAG'] = decoded[3]
ptable.loc[i, 'SHARK'] = decoded[0]
ptable.loc[i, 'YFT'] = decoded[2]
ptable.loc[i, 'NoF'] = 0.123081
ptable.loc[i, 'OTHER'] = 0.079142
i += 1
for j, file in enumerate(tqdm(glob('.. /input/fish-data/test_stg2/test_stg2/*.jpg'))):
bn = os.path.basename(file)
 bn = "test_stg2/" + bn
_, sample_img = load_img(file)
 pred = model.predict(np.array([sample_img,])) [0]
decoded = decode_prediction(pred, conf_thr=0.015)
 decoded = non_max_suppression(decoded,
max_output_size=1,
iou_threshold=0.5)
 decoded = decoded[0]['labels']
ptable.loc[i + j, 'image'] = bn
ptable.loc[i + j, 'ALB'] = decoded[5]
ptable.loc[i + j, 'BET'] = decoded[1]
ptable.loc[i + j, 'DOL'] = decoded[4]
ptable.loc[i + j, 'LAG'] = decoded[3]
ptable.loc[i + j, 'SHARK'] = decoded[0]
ptable.loc[i + j, 'YFT'] = decoded[2]
ptable.loc[i + j, 'NoF'] = 0.123081
ptable.loc[i + j, 'OTHER'] = 0.079142
return ptable<save_to_csv>
import seaborn as sns
pred_table = make_predictions()
pred_table.to_csv("neto_submit.csv", index=False)
 print(os.listdir("./"))<import_modules>
train[train['target'] <= 4.5]
import numpy as np
import pandas as pd
import six
import time
from random import randint
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from skimage.transform import resize
from keras import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.models import load_model
from keras.layers import Input,Dropout,BatchNormalization,Activation,Add
from keras.preprocessing.image import load_img
from keras.optimizers import Adam
from keras.utils.vis_utils import plot_model
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, concatenate, Dropout,BatchNormalization
from keras.layers import Conv2D, Concatenate, MaxPooling2D
from keras.layers import UpSampling2D, Dropout, BatchNormalization
from tqdm.notebook import tqdm
from keras import initializers
from keras import regularizers
from keras import constraints
from keras.utils import conv_utils
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.engine import InputSpec
from keras import backend as K
from keras.regularizers import l2
from keras import optimizers
from keras.engine.topology import Input
from keras.engine.training import Model
from keras.layers.convolutional import Conv2D, UpSampling2D, Conv2DTranspose
from keras.layers.core import Activation, SpatialDropout2D
from keras.layers.merge import concatenate,add
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
import tensorflow as tf
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.model_selection import KFold<load_from_disk>
train.drop([241352, 284103, 300936, 307139, 355831], axis = 0, inplace=True)
 train[train['target'] <= 4.5]
! unzip -q.. /input/tgs-salt-identification-challenge/train.zip -d train/
! unzip -q.. /input/tgs-salt-identification-challenge/test.zip -d test/<load_from_csv>
target = train['target']
train.drop('target', axis=1, inplace=True )
train = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv")
 depths = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv")
 sub = pd.read_csv(".. /input/tgs-salt-identification-challenge/sample_submission.csv")
 image_path = "/kaggle/working/train/images/"
mask_path = "/kaggle/working/train/masks/"
image_test_path = "/kaggle/working/test/images/"
model_path = 'unet_model.model'
submission_file = 'submission.csv'
t_start = time.time()
img_size = 101
epochs = 50
batch_size = 64<merge>
X_train, X_valid, y_train, y_valid = train_test_split(train, target, test_size=0.15, random_state=0 )
train = train[['id']].join(depths.set_index('id'), on='id')
 test = sub[['id']].join(depths.set_index('id'), on='id' )<feature_engineering>
lgbm = LGBMRegressor(tree_method='gpu_hist',learning_rate=0.07, max_depth=15, random_state=0, n_estimators=2000, n_jobs=-1)
 lgbm.fit(X_train, y_train, eval_set=[(X_valid, y_valid)])
 lgbm.score(X_valid, y_valid )
train["images"] = [np.array(load_img(image_path+"{}.png".format(idx), color_mode="grayscale"), dtype=np.uint8)/ 255 for idx in tqdm(train.id)]
train["masks"] = [np.array(load_img(mask_path+"{}.png".format(idx), color_mode="grayscale"), dtype=np.uint8)/ 255 for idx in tqdm(train.id)]<feature_engineering>
xgb = XGBRegressor(learning_rate=0.005, max_depth=8, n_estimators=6000, n_jobs=-1, tree_method='gpu_hist', random_state=3)
 xgb.fit(X_train, y_train, eval_set=[(X_valid, y_valid)])
 xgb.score(X_valid, y_valid )
train["coverage"] = train.masks.map(np.sum)/ pow(img_size, 2 )<categorify>
ensemble = VotingRegressor(estimators=[("xgb", xgb),("lgbm", lgbm)], weights=[1.1,1])
 ensemble.fit(X_train, y_train)
 ensemble.score(X_valid, y_valid )
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train["coverage_class"] = train.coverage.map(cov_to_class )<split>
model = ensemble.fit(train, target)
 model.score(train, target )
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cover_train, cover_test, depth_train, depth_test = train_test_split(
 train.id.values,
np.array(train.images.tolist() ).reshape(-1,img_size, img_size, 1),
np.array(train.masks.tolist() ).reshape(-1, img_size, img_size, 1),
train.coverage.values,
train.z.values,
test_size = 0.1 , stratify=train.coverage_class, random_state=42 )<concatenate>
pred = model.predict(test)
 pred
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0 )<define_search_model>
id = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')['id']
<choose_model_class><EOS> 
output = pd.DataFrame({'id': id, 'target': pred})
 output.to_csv("submission.csv", index=False)
 print("saved" )