Kaggle competition: AILAB ML Training #1
<SOS> metric: categorizationaccuracy Kaggle data source: ailab-ml-training-#1<categorify>
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl
df_dummy = pd.get_dummies(df_all['Neighborhood'],prefix='Neighborhood',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Neighborhood']<categorify>
%matplotlib inline
pd.set_option('max_rows', 1000)
 pd.set_option('max_columns', 1000 )
df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna('Other')
 df_dummy = pd.get_dummies(df_all['Exterior2nd'],prefix='Exterior2nd',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Exterior2nd']<categorify>
def seed_everything(seed=42):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 def get_strategy() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU {}'.format(tpu.cluster_spec().as_dict() ['worker']))
except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print('REPLICAS: {}'.format(strategy.num_replicas_in_sync))
return strategy
df_dummy = pd.get_dummies(df_all['Utilities'],prefix='Utilities',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Utilities']<categorify>
INPUT_DIR = '.. /input/ailab-ml-training-1/'
ARTIFACT_DIR = '.. /input/v2-5-ailab1-training/'
PATH = {
'train': os.path.join(INPUT_DIR, 'train.csv'),
'submission': os.path.join(INPUT_DIR, 'sample_submission.csv'),
'train_image_dir': os.path.join(INPUT_DIR, 'train_images/train_images'),
'test_image_dir': os.path.join(INPUT_DIR, 'test_images/test_images'),
'f0_weight': os.path.join(ARTIFACT_DIR, 'f0_best_weight.h5'),
'f1_weight': os.path.join(ARTIFACT_DIR, 'f1_best_weight.h5'),
'f2_weight': os.path.join(ARTIFACT_DIR, 'f2_best_weight.h5'),
'f3_weight': os.path.join(ARTIFACT_DIR, 'f3_best_weight.h5'),
'f4_weight': os.path.join(ARTIFACT_DIR, 'f4_best_weight.h5'),
'gcs': 'v2-ailab1-tfrecord-dataset'
}
ID = 'fname'
TARGET = 'label'
df_dummy = pd.get_dummies(df_all['LandContour'],prefix='LandContour',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['LandContour']<categorify>
seed_everything(args.seed )
df_dummy = pd.get_dummies(df_all['Condition1'],prefix='Condition1',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Condition1']<categorify>
train_df = pd.read_csv(PATH['train'])
 submission_df = pd.read_csv(PATH['submission'] )
def Condition2(c):
if c == 'Norm':
return 1
elif c == 'Feedr':
return 2
elif c == 'PosA':
return 3
elif c == 'PosN':
return 4
elif c == 'Artery':
return 5
else:
return 0<data_type_conversions>
def normalize(image):
image -= tf.constant([0.485 * 255, 0.456 * 255, 0.406 * 255])
 image /= tf.constant([0.229 * 255, 0.224 * 255, 0.225 * 255])
 return image
def get_model(
 input_size=(224, 224, 3),
backbone='efficientnet-b0',
weights='imagenet',
n_classes=10, 
):
print(f'Using backbone {backbone} and weights {weights}')
 model_fn = getattr(efn, f'EfficientNetB{backbone[-1]}')
 x = L.Input(shape=input_size, dtype='float32')
 y = normalize(x)
 y = model_fn(input_shape=input_size, weights=weights, include_top=False )(y)
 y = L.GlobalAveragePooling2D()(y)
 y = L.Dense(n_classes, activation='softmax' )(y)
 model = tf.keras.Model(x, y)
 return model
def read_tfrecords(tfrec, input_size=(224, 224, 3)) :
formats = {
'fname': tf.io.FixedLenFeature([], tf.string),
'image': tf.io.FixedLenFeature([], tf.string),
'label': tf.io.FixedLenFeature([], tf.int64)
 }
tfrec = tf.io.parse_single_example(tfrec, formats)
 image = tf.image.decode_image(tfrec['image'])
 image = tf.reshape(image,(input_size[0], input_size[1], 1))
image = tf.repeat(image, repeats=3, axis=2)
 image = tf.cast(image, tf.float32)
 return tfrec['fname'], image
def get_test_dataset(tfrec_filenames, args, ordered=True):
AUTO = tf.data.experimental.AUTOTUNE
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
func_sample = [
lambda e: read_tfrecords(e, input_size=(args.imsize, args.imsize, 3)) ,
]
ds = tf.data.TFRecordDataset(tfrec_filenames, num_parallel_reads=AUTO)
 ds = ds.map(func_sample[0], num_parallel_calls=AUTO)
 return ds
df_all['Condition2'] = df_all['Condition2'].apply(Condition2 ).astype(int )<categorify>
oof = np.zeros(( len(train_df), args.n_classes))
predictions = np.zeros(( len(submission_df), args.n_classes))
train_fnames = train_df[ID].tolist()
submission_fnames = submission_df[ID].tolist()
gcs_path = KaggleDatasets().get_gcs_path(PATH['gcs'])
 for fold in range(args.n_splits):
if fold not in args.use_folds:
print(f'skip fold {fold} training, because not in {args.use_folds}')
 continue
strategy = get_strategy()
with strategy.scope() :
model = get_model(
 input_size=(args.imsize, args.imsize, 3),
backbone=args.backbone,
weights=None,
n_classes=args.n_classes 
)
 model.load_weights(PATH[f'f{fold}_weight'])
 print('load weight {}'.format(PATH[f'f{fold}_weight']))
tfrecs = tf.io.gfile.glob(os.path.join(gcs_path, f'records/train_f{fold}_*_*.tfrec'))
print(f'inference oof {tfrecs}')
 ds = get_test_dataset(tfrecs, args, ordered=True)
 num_val_samples = sum(int(re.findall(r'\d+', fname)[-1])for fname in tfrecs)
 ds_fnames = ds.map(lambda fname, image: fname ).batch(num_val_samples)
 ds_images = ds.map(lambda fname, image: image ).batch(args.batch_size)
 _fnames = [fn.decode('utf-8')for fn in next(iter(ds_fnames)).numpy().tolist() ]
indices = [train_fnames.index(fn)for fn in _fnames]
oof[indices] = model.predict(ds_images)
 tfrecs = tf.io.gfile.glob(os.path.join(gcs_path, f'records/test_f0_*_*.tfrec'))
print(f'inference test {tfrecs}')
 ds = get_test_dataset(tfrecs, args, ordered=True)
 ds_fnames = ds.map(lambda fname, image: fname ).batch(len(submission_df))
ds_images = ds.map(lambda fname, image: image ).batch(args.batch_size)
 _fnames = [fn.decode('utf-8')for fn in next(iter(ds_fnames)).numpy().tolist() ]
indices = [submission_fnames.index(fn)for fn in _fnames]
predictions[indices] += model.predict(ds_images)/ args.n_splits
df_dummy = pd.get_dummies(df_all['BldgType'],prefix='BldgType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BldgType']<categorify>
score = accuracy_score(train_df[TARGET].values, np.argmax(oof, axis=1))
print(f'CV score: {score:.5f}' )
df_dummy = pd.get_dummies(df_all['HouseStyle'],prefix='HouseStyle',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['HouseStyle']<categorify>
np.save('oof', oof)
 np.save('predictions', predictions)
 submission_df[TARGET] = np.argmax(predictions, axis=1 ).tolist()
submission_df.to_csv('submission.csv', index=False)
 FileLink('submission.csv' )