Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<train_model>
input_path = '/kaggle/input/severstal-steel-defect-detection/'
base = '/kaggle/input/severstal-inference-base'
requirements_dir = base + '/requirements/'
cb.fit(x_train,y_train)
<predict_on_test>
!pip -q config set global.disable-pip-version-check true
!pip -q install {requirements_dir}Keras_Applications-1.0.8-py3-none-any.whl
!pip -q install {requirements_dir}efficientnet-1.1.1-py3-none-any.whl
y_pred=cb.predict(x_test )<import_modules>
!cp -r {base}/tpu_segmentation./
!cp -r {base}/*.py./
!rm -r tpu_segmentation *.py
AUTO = tf.data.experimental.AUTOTUNE
strategy = tf.distribute.get_strategy()
start_notebook = time()
print('Notebook started at: ', current_time_str())
 print('Tensorflow version: ', tf.__version__ )
from sklearn.metrics import r2_score<compute_test_metric>
IMAGE_SIZE =(256, 1600)
 target_size =(128, 800)
 input_shape =(*target_size, 3)
 N_CLASSES = 4
print('r2 score is:',r2_score(y_test,y_pred))<predict_on_test>
test_fnames = tf.io.gfile.glob(input_path + 'test_images/*')
 test_ids = [x.split('/')[-1].split('.')[0] for x in test_fnames]
get_test_path = lambda x: input_path + 'test_images/' + x + '.jpg'
y_pred_final=cb.predict(test )<feature_engineering>
def normalize_and_reshape(img, target_size):
img = tf.image.resize(img, target_size)
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.reshape(img, [*target_size, 3])
 return img
def get_image_and_id(file_name, target_size):
img = tf.io.read_file(file_name)
 img = tf.image.decode_jpeg(img, channels=3)
 img = normalize_and_reshape(img, target_size)
 img_id = tf.strings.split(file_name, os.path.sep)[-1]
img_id = tf.strings.split(img_id, '.')[0]
return img, img_id
def get_test_dataset(fnames, target_size, batch_size):
dataset = tf.data.Dataset.from_tensor_slices(fnames)
 dataset = dataset.map(lambda file_name: get_image_and_id(file_name, target_size), num_parallel_calls=AUTO)
 dataset = dataset.batch(batch_size=batch_size, drop_remainder=False)
 dataset = dataset.prefetch(AUTO)
 return dataset
sample_submission['winPlacePerc']=y_pred_final<save_to_csv>
df = pd.read_csv(base + '/weights_meta.csv')
 df1 = df[df.source == 1]
df2 = df[df.source == 2]
bin1 = df1[df1.type == 'bin']
bin1 = get_best_weights(bin1, 1)
 seg1 = df1[df1.type != 'bin']
seg1 = get_best_weights(seg1, 1)
 bin2 = df2[df2.type == 'bin']
seg2 = df2[df2.type != 'bin']
bin_weights = list(bin2.filename)+ list(bin1.filename)
 seg_weights = list(seg2.filename)+ list(seg1.filename )
sample_submission.to_csv('submission_pubg.csv',index=False )<import_modules>
!mkdir -p weights
!unzip -q {base}/mixed_weights.zip -d weights
!unzip -q {base}/binary_weights.zip -d weights
!unzip -q {base}/segmentation_weights.zip -d weights
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
pd.set_option('display.max_rows', 1100)
 pd.set_option('display.max_columns', 100 )<load_from_csv>
fnames_bin = [get_test_path(i)for i in test_ids]
test_dataset_bin = get_test_dataset(fnames_bin, target_size=(128, 800), batch_size=16)
 num_batches = tf.data.experimental.cardinality(test_dataset_bin);
print('num of batches', num_batches.numpy() )
DATA_FOLDER = '/kaggle/input/competitive-data-science-predict-future-sales/'
train = pd.read_csv(os.path.join(DATA_FOLDER, 'sales_train.csv'))
items = pd.read_csv(os.path.join(DATA_FOLDER, 'items.csv'))
item_categories = pd.read_csv(os.path.join(DATA_FOLDER, 'item_categories.csv'))
shops = pd.read_csv(os.path.join(DATA_FOLDER, 'shops.csv'))
sample_sub = pd.read_csv(os.path.join(DATA_FOLDER, 'sample_submission.csv'))
test = pd.read_csv(os.path.join(DATA_FOLDER, 'test.csv'))
cities = pd.read_csv('.. /input/shops-en/shops-en.csv' )<feature_engineering>
ensemble_outputs = []
with strategy.scope() :
X = L.Input(shape=input_shape)
 for i, w in enumerate(bin_weights):
base_name = w.split('-bin')[0]
model = build_classifier(base_name, n_classes = 1, input_shape=input_shape, weights = None, name_suffix='-M{}'.format(i+1))
model.load_weights('weights/' + w)
 model_output = model(X)
 ensemble_outputs.append(model_output)
 Y = L.Average()(ensemble_outputs)
 binary_ensemble = tf.keras.Model(inputs=X, outputs=Y, name='Binary_Classification_Ensemble')
 binary_ensemble.compile(optimizer='adam', loss='binary_crossentropy', metrics=[])
 del model, ensemble_outputs, model_output
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1200]
median = np.median(train[train['item_id']==2973].item_price)
 train.loc[train['item_price']<0,'item_price'] = median<feature_engineering>
start_preds = time()
binary_predictions = binary_ensemble.predict(test_dataset_bin)
 del binary_ensemble
K.clear_session()
gc.collect()
print('Elapsed time(binary predictions){}'.format(time_passed(start_preds)) )
cities['city'] = cities['shop_name'].str.split(" " ).map(lambda x: x[1])
 cities.iloc[20,2] = "Moscow"
cities.iloc[42,2] = "St.Petersburg"
cities.iloc[43,2] = "St.Petersburg"<data_type_conversions>
ensemble_outputs = []
with strategy.scope() :
X = L.Input(shape=input_shape)
 for i, w in enumerate(seg_weights):
backbone_name = w.split('-unetpp')[0]
model = xnet(backbone_name, num_classes = 4, input_shape=input_shape, weights = None)
 model._name = '{}-M{}'.format(model.name, i+1)
 model.load_weights('weights/' + w)
 model_output = model(X)
 ensemble_outputs.append(model_output)
 Y = L.Average()(ensemble_outputs)
 seg_ensemble = tf.keras.Model(inputs=X, outputs=Y, name='Mask_Segmentation_Ensemble')
 seg_ensemble.compile(optimizer='adam', loss='binary_crossentropy', metrics=[])
 del model, ensemble_outputs, model_output
!rm -r weights
def optimize_memory(df):
start_mem = df.memory_usage().sum() / 1024**2
integers = ['int8','int16','int32','int64']
floats = ['float32','float64']
int_cols = [c for c in df if df[c].dtype in integers]
float_cols = [c for c in df if df[c].dtype in floats]
for i in int_cols:
df[i] = pd.to_numeric(df[i], downcast='integer')
 for i in float_cols:
df[i] = pd.to_numeric(df[i], downcast='float')
 end_mem = df.memory_usage().sum() / 1024**2
print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<drop_column>
THRESHOLD = 0.80
masked_indexes = np.where(binary_predictions>=THRESHOLD)[0]
unmasked_indexes = np.where(binary_predictions<THRESHOLD)[0]
seg_ids = list(np.array(test_ids)[masked_indexes])
 no_seg_ids = list(np.array(test_ids)[unmasked_indexes])
 print(len(seg_ids), len(no_seg_ids), len(seg_ids)+ len(no_seg_ids), len(test_ids))
test['date_block_num'] = 34
df_sales = pd.concat([df_sales,test], ignore_index = True)
 df_sales.drop('ID',axis = 1, inplace = True)
 df_sales.fillna(0,inplace = True )<merge>
fnames_seg = [get_test_path(i)for i in seg_ids]
batch_size = 8
test_dataset_seg = get_test_dataset(fnames_seg, target_size=target_size, batch_size=batch_size)
 num_batches = tf.data.experimental.cardinality(test_dataset_seg);
print('num of batches', num_batches.numpy() )
df_sales = pd.merge(df_sales,items,how = 'left',on = 'item_id' ).drop('item_name', axis = 1)
 df_sales = pd.merge(df_sales,cities,how = 'left',on = 'shop_id' ).drop('shop_name', axis = 1)
 df_sales['city_code'] = LabelEncoder().fit_transform(df_sales['city'])
 df_sales.drop('city',axis = 1,inplace = True )<categorify>
n_batches = 1
sample_preds = seg_ensemble.predict(test_dataset_seg.take(n_batches))
examples = retrieve_examples(test_dataset_seg, batch_size*n_batches)
 idx = -1
mask_rgb = [(230, 184, 0),(0, 128, 0),(102, 0, 204),(204, 0, 102)]
df_sales['month'] =df_sales['date_block_num'] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 df_sales['days'] = df_sales['month'].map(days)
 holidays = pd.Series([8,1,1,0,2,1,0,0,0,0,1,0])
 df_sales['holidays'] = df_sales['month'].map(holidays )<drop_column>
thresh_upper = [0.7,0.7,0.7,0.7]
thresh_lower = [0.4,0.5,0.4,0.5]
min_area = [180, 260, 200, 500]
empty_mask = np.zeros(target_size, int)
 rles_dict = {}
for img_prefix in no_seg_ids:
for c in range(N_CLASSES):
row_name = '{}.jpg_{}'.format(img_prefix, c+1)
 rles_dict[row_name] = ''
start_preds = time()
for item in test_dataset_seg:
mask_predictions = seg_ensemble.predict(item[0])
 for k, p in enumerate(mask_predictions):
for ch in range(N_CLASSES):
ch_probs = p[..., ch]
ch_pred =(ch_probs > thresh_upper[ch])
 if ch_pred.sum() < min_area[ch]:
ch_pred = empty_mask.copy()
else:
ch_pred =(ch_probs > thresh_lower[ch])
 mask_predictions[k,:,:,ch] = ch_pred
img_ids = item[1]
ids = [l.decode('utf-8')for l in img_ids.numpy() ]
rles = [create_rles(p, IMAGE_SIZE)for p in mask_predictions]
for i in range(len(ids)) :
rle = rles[i]
img_prefix = ids[i]
for c in range(N_CLASSES):
row_name = '{}.jpg_{}'.format(img_prefix, c+1)
 rles_dict[row_name] = rle[c].numpy().decode('utf-8')
 print('Elapsed time(mask predictions){}'.format(time_passed(start_preds)) )
<categorify><EOS> 
df = pd.DataFrame.from_dict(rles_dict, orient='index')
 df.reset_index(level=0, inplace=True)
 df.columns = ['ImageId_ClassId', 'EncodedPixels']
df.to_csv('submission.csv', index=False )