Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<merge>
%matplotlib inline
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True )<merge>
PadIfNeeded,
HorizontalFlip,
VerticalFlip,
CenterCrop,
Crop,
Compose,
Transpose,
RandomRotate90,
ElasticTransform,
GridDistortion,
OpticalDistortion,
RandomSizedCrop,
OneOf,
CLAHE,
RandomBrightnessContrast,
RandomGamma,
RandomContrast,
InvertImg,
RandomContrast
)
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True )<merge>
os.listdir('/kaggle/input/severstal-steel-defect-detection')
 train_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/train.csv')
 train_df['Image_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[0])
 train_df['class_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[1])
 train_df['mask']= ~train_df[train_df.columns[1]].isna()
print(train_df.shape)
 train_df.head(5 )
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True )<merge>
mask_count=train_df.groupby(by='Image_id' ).agg(np.sum ).reset_index()
print(mask_count.shape)
 mask_count.head(3 )
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True )<merge>
mask_count.sort_values(by='mask',ascending=False,inplace=True)
 mask_count.head()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True )<merge>
sub_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id']=sub_df[sub_df.columns[0]].apply(lambda x : x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() ,columns=['Image_id'])
 print(test_imgs.shape)
 test_imgs.head()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True )<merge>
train_path = '/kaggle/input/severstal-steel-defect-detection/train_images/'
train_dir=os.listdir(train_path)
 train_dir[:3]
test_path = '/kaggle/input/severstal-steel-defect-detection/test_images/'
test_dir = os.listdir(test_path )
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True )<merge>
def mask2rle(img):
pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape):
depth = len(rles)
 masks = np.zeros(( *input_shape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle, input_shape)
 return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True )<filter>
pa='/kaggle/input/efficientnet'
sys.path.append(os.path.abspath('/kaggle/input/efficientnet-models/efficientnet-master/'))
N_CLASSES=4
def ClsModel(n_classes=1, input_shape=(254,600,3)) :
base_model = EfficientNetB2(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(5,5), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1600, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model = ClsModel(n_classes=N_CLASSES)
 N_CLASSES=4
def ClsModel(n_classes=1, input_shape=(254,254,3)) :
base_model = EfficientNetB3(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(3,3), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1024, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model_class = ClsModel(n_classes=N_CLASSES)
matrix = matrix[matrix.date_block_num > 11]<correct_missing_values>
class DataGenerator1(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(600,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix )<create_dataframe>
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 254), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(254,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
data = matrix.copy()<prepare_x_and_y>
model_class_2 = load_model('/kaggle/input/efficent-model-and-weights-975/efficent_class_model.h5')
X = data[data.date_block_num <= 33].drop(['item_cnt_month'], axis=1)
 y = data[data.date_block_num <= 33]['item_cnt_month']
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<drop_column>
all_classifer_path=['/kaggle/input/efficent-b2-600/EfficientNetB3_efficenent_254_600_exp_4.h5',
'/kaggle/input/efficient977/EfficientNetB3_efficenent_exp_2.h5']
class_models =[model,model_class]
pred_class_i=[]
pred=[]
DataGenerator=[DataGenerator1,DataGenerator2]
for i in range(2):
augmentaion=[1,HorizontalFlip,VerticalFlip]
for aug in augmentaion:
class_models[i].load_weights(all_classifer_path[i])
 test_generator = DataGenerator[i](
 test_imgs.index,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
augmentaion=[aug],
batch_size=1,
n_classes=4 
)
 batch_pred_class = class_models[i].predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 pred.append(batch_pred_class.round().astype(int))
del matrix
del sales_train_df
del train
del shops_df
del shops
del cats
del item_categories_df
del items_df
del items<compute_train_metric>
test=pred.copy()
reg = LinearRegression()
reg.fit(X_train,y_train)
 print('Linear Regression RMSE:', mean_squared_error(y_valid, reg.predict(X_valid), squared = False))<import_modules>
test_1=np.array(test)
 test_1=test_1[0]+test_1[1]+test_1[2]+test_1[3]+test_1[4]+test_1[5]
test_1[np.where(test_1<=2)]=0
test_1[np.where(test_1>2)]=1
np.sum(test_1,axis=0 )
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV<train_on_grid>
np.sum(test_1.round().astype(int),axis=0 )
<train_model>
aaa=[128/43/741/120]
reg_ridge = Ridge(alpha = 1000)
 reg_ridge.fit(X_train,y_train)
 print('Ridge RMSE:', mean_squared_error(y_valid, reg_ridge.predict(X_valid), squared = False))<predict_on_test>
np.sum(batch_pred_class )
reg_ridge.fit(X, y)
 Y_test = reg_ridge.predict(X_test ).clip(0, 20)
 submissionRidge = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
} )<import_modules>
pred_class =test_1.round().astype(int )
from sklearn.model_selection import train_test_split<import_modules>
np.sum(pred_class,axis=0 )
from sklearn.model_selection import train_test_split<import_modules>
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,
batch_size=1, dim=(256, 1600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
img_path = f"{self.base_path}/{im_name}"
img =cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
 img_i[i,]=img
return img_i
def __generate_y(self, ID):
img_i = np.empty(( self.batch_size,*self.dim,self.n_classes),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
img =build_masks(rles, input_shape=self.dim)
 img_i[i,]=img
return img_i
def __check_input__(self,index):
X,y=self.__getitem__(index)
 indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
ID=list_IDs_batch[0]
X=X[0]
y=y[0]
im_name=self.df['Image_id'].loc[ID]
img= cv2.imread(train_path+im_name)
 image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 fig,axis = plt.subplots(nrows=1,ncols=1, figsize=(20, 20))
for i in range(1):
try:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis[i].imshow(img_i)
 except IndexError:
print("IndexError : so skip")
 img_i=mask_image(X,y)
 axis.imshow(img_i,cmap='gray')
 break
except TypeError:
img_i=mask_image(X,y)
 print("i")
 axis.imshow(img_i,cmap='gray')
 print("TypeError(axis[i] is fails): so skip")
 break
img_i=mask_image(img.astype(np.uint8),masks.astype(np.uint8))
plt.figure(figsize=(10,10))
plt.imshow(img_i )
from lightgbm import LGBMRegressor<train_model>
non_missing_train_idx = mask_count[mask_count['mask'] > 0]
lgbm_model = LGBMRegressor(
 n_jobs=-1,
learning_rate=0.2,
max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42 
)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.33, random_state=7)
 eval_set = [(X_valid, y_valid)]
lgbm_model.fit(
 X_train,
y_train,
eval_metric="rmse",
eval_set=eval_set,
verbose=True,
early_stopping_rounds = 3 )<predict_on_test>
BATCH_SIZE = 10
train_idx, val_idx = train_test_split(
 non_missing_train_idx.index,
random_state=25,
test_size=0.2 
)
 train_generator = DataGenerator(
 train_idx,
df=non_missing_train_idx,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4,
mode='fit' 
)
 val_generator = DataGenerator(
 val_idx,
df=non_missing_train_idx,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4 
)
 train_generator.__check_input__(0 )
Y_test = lgbm_model.predict(X_test ).clip(0, 20)
 submissionLGBM = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
} )<split>
def dice_coef(y_true, y_pred, smooth= 1e-6):
y_pred = K.round(y_pred)
 intersection = K.sum(y_true * y_pred, axis=(1,2))
return K.mean(( 2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth))
def dice_loss(y_true, y_pred,smooth = 1e-9):
intersection = K.sum(y_true * y_pred, axis=(1,2))
score =(2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth)
 return K.mean(1-score)
 def bce_dice_loss(y_true, y_pred):
return dice_loss(y_true, y_pred )
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.33, random_state=7 )<import_modules>
def dice_coef(y_true, y_pred, smooth= 1e-6):
y_pred = K.round(y_pred)
 intersection = K.sum(y_true * y_pred,axis=(1,2))
return K.mean(( 2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth))
opt=keras.optimizers.Adam(lr=1e-4)
import tensorflow as tf
from tensorflow.keras.layers.experimental import preprocessing
from keras.layers import LSTM, Dense, Dropout
from keras.models import Sequential<choose_model_class>
model_seg = load_model('/kaggle/input/seg-model/seg_model.h5')
 model_seg.load_weights('/kaggle/input/16000-model/1600.h5' )
tf.keras.backend.clear_session()
model = Sequential()
model.add(LSTM(units=64, input_shape=(35, 1)))
 model.add(Dropout(0.3))
model.add(Dense(1))
model.compile(loss='mse',
optimizer='adam',
metrics=['mean_squared_error'])
 model.summary()<train_model>
np.sum(pred_class,axis=0 )
X_train = np.expand_dims(X_train.values,axis = 2)
 X_test = np.expand_dims(X_test.values,axis = 2)
 X_valid = np.expand_dims(X_valid.values,axis = 2)
 print(X_train.shape, X_test.shape )<train_model>
sub_df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id'] = sub_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() , columns=['Image_id'])
 test_imgs=test_imgs
filtered_mask = sub_df['Image_id'].isin(test_imgs["Image_id"].values)
 filtered_sub_df = sub_df[filtered_mask].copy()
null_sub_df = sub_df[~filtered_mask].copy()
null_sub_df['EncodedPixels'] = null_sub_df['EncodedPixels'].apply(
 lambda x: ' ')
 filtered_sub_df.reset_index(drop=True, inplace=True)
 test_imgs.reset_index(drop=True, inplace=True)
 print(filtered_sub_df.shape)
 print(null_sub_df.shape)
 filtered_sub_df.head()
<import_modules>
all_class=pred_class.flatten()
all_class
filtered_sub_df['EncodedPixels']=all_class
from keras.utils import plot_model<save_to_csv>
def sliding_window(img, sz=256):
total=7
x=0
img=img[0]
new_img = np.empty(( total,256,256,3),dtype=np.uint8)
 for count in range(total-1):
new_img[count,]=img[:, x:x+sz, :]
x=x+sz
new_img[-1,]=img[:, -256:, :]
return new_img
submission_df['item_cnt_month'] =(0.1*submissionRidge['item_cnt_month'] + 0.9*submissionLGBM['item_cnt_month'])
 submission_df.to_csv('submission.csv', index=False)
 print('output file :','submission','saved' )<import_modules>
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,
batch_size=1, dim=(256, 1600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)// self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return crop_rand_pattern(X, y)
 elif self.mode == 'predict':
return sliding_window(X)
 elif self.mode=='val':
y = self.__generate_y(list_IDs_batch)
 return sliding_window_predict(X, y)
 else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
img_path = f"{self.base_path}/{im_name}"
img =cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
 img_i[i,]=img
return img_i
def __generate_y(self, ID):
img_i = np.empty(( self.batch_size,*self.dim,self.n_classes),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
img =build_masks(rles, input_shape=self.dim)
 img_i[i,]=img
return img_i
def __check_input__(self,index):
X,y=self.__getitem__(index)
 indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
ID=list_IDs_batch
X=X
y=y
for i,ID in enumerate(list_IDs_batch):
im_name=self.df['Image_id'].loc[ID]
img= cv2.imread(train_path+im_name)
 image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 img_i=mask_image(img.astype(np.uint8),masks.astype(np.uint8))
plt.figure(figsize=(10,10))
plt.imshow(img_i)
 plt.show()
col=y.shape[0]
fig,axis = plt.subplots(nrows=1,ncols=col, figsize=(20, 20))
for i in range(col):
try:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis[i].imshow(img_i)
 except IndexError:
print("IndexError : so skip")
 img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis.imshow(img_i,cmap='gray')
 break
except TypeError:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
print("i")
 axis.imshow(img_i,cmap='gray')
 print("TypeError(axis[i] is fails): so skip")
 break
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import RandomizedSearchCV,cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import make_scorer
from itertools import product
import re
import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords 
<install_modules>
model_seg_2 = load_model('/kaggle/input/seg-model-256-256/seg_model.h5')
 model_seg_2.load_weights('/kaggle/input/seg-weights-898/bce_dice_loss_bce_from_8963_exp_4.h5' )
!pip install pymorphy2
!pip install pymorphy2-dicts
!pip install DAWG-Python<import_modules>
test_df = []
for i in range(0, test_imgs.shape[0], 300):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 300)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model_seg.predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 for j, b in tqdm(enumerate(batch_idx)) :
filename = test_imgs['Image_id'].iloc[b]
image_df = filtered_sub_df[filtered_sub_df['Image_id'] == filename].copy()
pred_masks=batch_pred_masks[j,].round().astype(int)
 value = np.sum(pred_masks,axis=(1,0))
c1=np.array(value*image_df['EncodedPixels'])
 pred_masks=batch_pred_masks[j,]
pred_masks[np.where(pred_masks>0.6)]=1
pred_masks[np.where(pred_masks<=0.6)]=0
pred_rles = build_rles(pred_masks)
 pred_rles=np.array(pred_rles)
 pred_rles[c1==0]=''
image_df['EncodedPixels'] = pred_rles
test_df.append(image_df)
 gc.collect()
import pymorphy2<load_from_csv>
test_df = pd.concat(test_df)
 print(test_df.shape)
 test_df.head()
sample_submissions=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 sales_train_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 items_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shops_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 item_categories_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv" )<set_options>
test_df[['ImageId_ClassId', 'EncodedPixels']].to_csv('submission.csv', index=False )
<merge><EOS> 
a=np.array([0,2])
 a[np.where(a>0.7)]=10
a