Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<merge>
%matplotlib inline
visit_data=air_visit_data.merge(date_info,left_on='visit_date',right_on='calendar_date',how='left')
 visit_data.drop('calendar_date',axis=1,inplace=True)
 visit_data['visitors']=visit_data.visitors.map(pd.np.log1p)
 visit_data.head(10 )<merge>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
except ValueError:
tpu = None
gpus = tf.config.experimental.list_logical_devices('GPU')
 if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU: " + str(tpu.master()))
elif len(gpus)> 0:
strategy = tf.distribute.MirroredStrategy(gpus)
 print("Running on ",len(gpus)," GPU(s)")
 else:
strategy = tf.distribute.get_strategy()
print("Running on CPU")
 print("Number of units: {}".format(strategy.num_replicas_in_sync))
sample_submission['air_store_id']=sample_submission.id.map(
 lambda x:'_'.join(x.split('_')[:-1]))
sample_submission['calendar_date']=sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors',axis=1,inplace=True)
 sample_submission=sample_submission.merge(date_info, on='calendar_date',how='left')
 sample_submission=sample_submission.merge(
 visitors, on=['air_store_id','day_of_week','holiday_flg'],how='left')
 sample_submission.head(10 )<count_missing_values>
AUTO = tf.data.experimental.AUTOTUNE
num_units = strategy.num_replicas_in_sync
if num_units == 8:
BATCH_SIZE = 16 * num_units
VALIDATION_BATCH_SIZE = 16 * num_units
start_lr = 0.00001
min_lr = 0.00001
max_lr = 0.00005 * num_units
rampup_epochs = 8
sustain_epochs = 0
exp_decay = 0.8
else:
BATCH_SIZE = 2
VALIDATION_BATCH_SIZE = 2
start_lr = 0.0003
min_lr = 0.000001
rampup_epochs = 8
sustain_epochs = 0
exp_decay = 0.7
missing_values_table(sample_submission )<merge>
BASE = '.. /input/prostate-cancer-grade-assessment/'
TEST_PATH = '.. /input/prostate-cancer-grade-assessment/test_images'
test_csv_path = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
missings=sample_submission.visitors.isnull()
sample_submission.loc[missings,'visitors']=sample_submission[missings].merge(
 visitors[visitors.holiday_flg==0],on=('air_store_id','day_of_week'), how='left')['visitors_y'].values
missing_values_table(sample_submission )<merge>
train = pd.read_csv(BASE + 'train.csv')
 test = pd.read_csv(BASE + 'test.csv' )
missings=sample_submission.visitors.isnull()
sample_submission.loc[missings,'visitors']=sample_submission[missings].merge(
 visitors[['air_store_id','visitors']].groupby('air_store_id' ).mean().reset_index() ,on='air_store_id',how='left')['visitors_y'].values
missing_values_table(sample_submission )<prepare_output>
image_paths = [BASE + 'train_images/' + x + '.tiff' for x in train['image_id'].values.tolist() ]
sample_submission['visitors']=sample_submission.visitors.map(pd.np.expm1)
 sample_submission=sample_submission[['id','visitors']]
final['visitors'][final['visitors']==0]=sample_submission['visitors'][final['visitors']==0]
sub_file=final.copy()
sub_file.head() 
<save_to_csv>
def read_image(image_path, resize_ratio=1):
if not(isinstance(image_path, str)) :
image_path = image_path.numpy().decode('utf-8')
 image_level_1 = skimage.io.MultiImage(image_path)[1]
if resize_ratio != 1:
new_w = int(image_level_1.shape[1]*resize_ratio)
 new_h = int(image_level_1.shape[0]*resize_ratio)
 image_level_1 = cv2.resize(
 image_level_1,(new_w, new_h), interpolation=cv2.INTER_AREA)
 return image_level_1
sub_file['visitors']=np.mean([final['visitors'],sample_submission['visitors']],axis=0)
 sub_file.to_csv('sub_math_mean_1.csv',index=False)
 sub_file['visitors']=(final['visitors']*sample_submission['visitors'])**(1/2)
 sub_file.to_csv('sub_geo_mean_1.csv',index=False)
 sub_file['visitors']=2/(1/final['visitors']+1/sample_submission['visitors'])
 sub_file.to_csv('sub_hrm_mean_1.csv',index=False)
<load_from_csv>
def _pad_image(image, pad_len, pad_val):

 if image is None:
return None
elif image.ndim == 2:
return np.pad(
 image,(( pad_len, pad_len),(pad_len, pad_len)) , pad_val)
 elif image.ndim == 3:
return np.pad(
 image,(( pad_len, pad_len),(pad_len, pad_len),(0, 0)) , pad_val)
 return None
def _transpose_image(image):

 if image is None:
return None
elif image.ndim == 2:
return np.transpose(image,(1, 0)).copy()
elif image.ndim == 3:
return np.transpose(image,(1, 0, 2)).copy()
return None
def _get_tissue_parts_indices(tissue, min_consec_info):

 split_points = np.where(np.diff(tissue)!= 1)[0]+1
tissue_parts = np.split(tissue, split_points)
 return [
tp for tp in tissue_parts if len(tp)>= min_consec_info
]
def _get_tissue_subparts_coords(subtissue, patch_size, min_decimal_keep):

 start, end = subtissue[0], subtissue[-1]
num_subparts =(end-start)/patch_size
if num_subparts % 1 < min_decimal_keep and num_subparts >= 1:
num_subparts = math.floor(num_subparts)
 else:
num_subparts = math.ceil(num_subparts)
 excess =(num_subparts*patch_size)-(end-start)
 shift = excess // 2
return [
i * patch_size + start - shift
for i in range(num_subparts)
 ]
def _eval_and_append_xy_coords(coords,
image,
mask,
patch_size,
x, y,
min_patch_info,
transposed,
precompute):

 patch_1d =(
 image[y: y+patch_size, x:x+patch_size, :] 
.mean(axis=2)
.reshape(-1)
)
 idx_tissue = np.where(patch_1d <= 210)[0]
idx_black = np.where(patch_1d < 5)[0]
idx_background = np.where(patch_1d > 210)[0]
if len(idx_tissue)> 0:
patch_1d[idx_black] = 210
patch_1d[idx_background] = 210
val1 = int(patch_1d.mean())
 val2 = mask[y:y+patch_size, x:x+patch_size].mean()
if val2 > min_patch_info:
if precompute:
if transposed:
coords = np.concatenate([
coords, [[val1, x-patch_size, y-patch_size]]
])
 else:
coords = np.concatenate([
coords, [[val1, y-patch_size, x-patch_size]]
])
 else:
coords = np.concatenate([
coords, [[val1, y, x]]
])
 return coords
def compute_coords(image,
patch_size=256,
precompute=False,
min_patch_info=0.35,
min_axis_info=0.35,
min_consec_axis_info=0.35,
min_decimal_keep=0.7 
):

 if type(image)!= np.ndarray:
image = image.numpy()
mask = _mask_tissue(image)
 coords = np.zeros([0, 3], dtype=int)
 image = _pad_image(image, patch_size, 'maximum')
 mask = _pad_image(mask, patch_size, 'minimum')
 y_sum = mask.sum(axis=1)
 x_sum = mask.sum(axis=0)
 if len(np.where(x_sum > 0)[0])> len(np.where(y_sum > 0)[0]):
image = _transpose_image(image)
 mask = _transpose_image(mask)
 y_sum, _ = x_sum, y_sum
transposed = True
else:
transposed = False
y_tissue = np.where(y_sum >=(patch_size*min_axis_info)) [0]
if len(y_tissue)< 1:
warnings.warn("Not enough tissue in image(y-dim)", RuntimeWarning)
 if precompute: return [(0, 0, 0)]
else: return image, [(0, 0, 0)]
y_tissue_parts_indices = _get_tissue_parts_indices(
 y_tissue, patch_size*min_consec_axis_info)
 if len(y_tissue_parts_indices)< 1:
warnings.warn("Not enough tissue in image(y-dim)", RuntimeWarning)
 if precompute: return [(0, 0, 0)]
else: return image, [(0, 0, 0)]
for yidx in y_tissue_parts_indices:
y_tissue_subparts_coords = _get_tissue_subparts_coords(
 yidx, patch_size, min_decimal_keep)
 for y in y_tissue_subparts_coords:
x_slice_sum = mask[y:y+patch_size, :].sum(axis=0)
 x_tissue = np.where(x_slice_sum >=(patch_size*min_axis_info)) [0]
x_tissue_parts_indices = _get_tissue_parts_indices(
 x_tissue, patch_size*min_consec_axis_info)
 for xidx in x_tissue_parts_indices:
x_tissue_subparts_coords = _get_tissue_subparts_coords(
 xidx, patch_size, min_decimal_keep)
 for x in x_tissue_subparts_coords:
coords = _eval_and_append_xy_coords(
 coords, image, mask, patch_size, x, y,
min_patch_info, transposed, precompute 
)
 if len(coords)< 1:
warnings.warn("Not enough tissue in image(x-dim)", RuntimeWarning)
 if precompute: return [(0, 0, 0)]
else: return image, [(0, 0, 0)]
if precompute:
return coords
else:
return image, coords
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'])
 for df in ['ar','hr']:
data[df]['reserve_visitors'] = np.log1p(data[df]['reserve_visitors'])
 data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_dow'] = data[df]['visit_datetime'].dt.dayofweek
data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 data[df]['early_reservation'] = data[df]['reserve_datetime_diff'] > 2
data[df]['late_reservation'] = data[df]['reserve_datetime_diff'] <= 2
tmp1 = data[df][data[df]['early_reservation']].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs1', 'reserve_visitors':'rv1'})
 tmp2 = data[df][data[df]['early_reservation']].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs2', 'reserve_visitors':'rv2'})
 tmp3 = data[df][data[df]['late_reservation']].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs12', 'reserve_visitors':'rv12'})
 tmp4 = data[df][data[df]['late_reservation']].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs22', 'reserve_visitors':'rv22'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'])
 data[df] = pd.merge(data[df], tmp3, how='left', on=['air_store_id','visit_date'])
 data[df] = pd.merge(data[df], tmp4, how='left', on=['air_store_id','visit_date'])
 data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['week'] = data['tra']['visit_date'].dt.week
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['week'] = data['tes']['visit_date'].dt.week
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
unique_stores = data['tes']['air_store_id'].unique()
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True)
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].max().rename(columns={'visitors':'max_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].count().rename(columns={'visitors':'count_observations'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 stores = pd.merge(stores, data['as'], how='left', on=['air_store_id'])
 stores['air_genre_name'] = stores['air_genre_name'].map(lambda x: str(str(x ).replace('/',' ')))
 stores['air_area_name'] = stores['air_area_name'].map(lambda x: str(str(x ).replace('-',' ')))
 lbl = preprocessing.LabelEncoder()
for i in range(10):
stores['air_genre_name'+str(i)] = lbl.fit_transform(stores['air_genre_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_area_name'+str(i)] = lbl.fit_transform(stores['air_area_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_genre_name'] = lbl.fit_transform(stores['air_genre_name'])
 stores['air_area_name'] = lbl.fit_transform(stores['air_area_name'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['day_of_week'] = lbl.fit_transform(data['hol']['day_of_week'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(train, stores, how='inner', on=['air_store_id','dow'])
 test = pd.merge(test, stores, how='left', on=['air_store_id','dow'])
 for df in ['ar','hr']:
train = pd.merge(train, data[df], how='left', on=['air_store_id','visit_date'])
 test = pd.merge(test, data[df], how='left', on=['air_store_id','visit_date'])
 train['id'] = train.apply(lambda r: '_'.join([str(r['air_store_id']), str(r['visit_date'])]), axis=1)
 train['total_reserv_sum'] = train['rv1_x'] + train['rv1_y']
train['total_reserv_mean'] =(train['rv2_x'] + train['rv2_y'])/ 2
train['total_reserv_dt_diff_mean'] =(train['rs2_x'] + train['rs2_y'])/ 2
test['total_reserv_sum'] = test['rv1_x'] + test['rv1_y']
test['total_reserv_mean'] =(test['rv2_x'] + test['rv2_y'])/ 2
test['total_reserv_dt_diff_mean'] =(test['rs2_x'] + test['rs2_y'])/ 2
train['date_int'] = train['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['date_int'] = test['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['var_max_lat'] = train['latitude'].max() - train['latitude']
train['var_max_long'] = train['longitude'].max() - train['longitude']
test['var_max_lat'] = test['latitude'].max() - test['latitude']
test['var_max_long'] = test['longitude'].max() - test['longitude']
train['lon_plus_lat'] = train['longitude'] + train['latitude']
test['lon_plus_lat'] = test['longitude'] + test['latitude']
lbl = preprocessing.LabelEncoder()
train['air_store_id2'] = lbl.fit_transform(train['air_store_id'])
 test['air_store_id2'] = lbl.transform(test['air_store_id'])
 col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date','visitors']]
train = train.fillna(-1)
 test = test.fillna(-1)
 train['visitors'] = np.log1p(train['visitors'].values )<define_variables>
def _patch_augment(patch, p=0.5):

 r1 = tf.random.uniform(
 shape=(4,), minval=0, maxval=1)
 r2 = tf.random.uniform(
 shape=() , minval=0, maxval=4, dtype=tf.int32)
 if r1[0] >(1-p):
patch = tf.image.rot90(patch, k=r2)
 if r1[1] >(1-p):
patch = tf.image.random_flip_left_right(patch)
 if r1[2] >(1-p):
patch = tf.image.random_flip_up_down(patch)
 if r1[3] >(1-p):
patch = tf.transpose(patch,(1, 0, 2))
return patch
def _excess_coords_filtering(coords, sample_size, proportion=0.25):

 if len(coords)> sample_size:
c = tf.transpose(coords)
 v = tf.gather(c, 0)
 num = tf.cast(len(v), tf.float32)
 sample_size = tf.cast(sample_size, tf.float32)
 indices_reduced = int(tf.math.ceil(
 num *(1 -(( num - sample_size)/ num)* proportion)
))
v_argsort = tf.argsort(v)
 indices = tf.gather(v_argsort, tf.range(indices_reduced))
indices = tf.sort(indices)
 coords = tf.gather(coords, indices)
 return coords
@tf.function
def patch_image(image, coords, sample_size=36, patch_size=256):
l = tf.cast(tf.math.sqrt(tf.cast(sample_size, tf.float32)) , tf.int32)
 coords = _excess_coords_filtering(coords, sample_size)
 if len(coords)< sample_size:
indices = tf.tile(
 tf.range(len(coords)) , [tf.math.ceil(sample_size/len(coords)) ])
 indices = indices[:sample_size]
else:
indices = tf.range(sample_size)
 coords = tf.gather(coords, indices)
 coords = tf.random.shuffle(coords)
 patched_image = tf.zeros(
 [0, patch_size, patch_size, 3], dtype=tf.dtypes.uint8)
 for i in range(sample_size):
y = tf.gather_nd(coords, [i, 1])
 x = tf.gather_nd(coords, [i, 2])
 shape = tf.shape(image)
 h = tf.gather(shape, 0)
 w = tf.gather(shape, 1)
 if y < 0: y = 0
if x < 0: x = 0
if y > h-patch_size: y = h-patch_size
if x > w-patch_size: x = w-patch_size
patch = tf.slice(
 image,
tf.stack([y, x, 0]),
tf.stack([patch_size, patch_size, -1]))
patched_image = tf.concat([
patched_image, tf.expand_dims(patch, 0)], axis=0)
 patched_image = tf.reshape(patched_image,(-1, patch_size*l, patch_size, 3))
patched_image = tf.transpose(patched_image,(0, 2, 1, 3))
patched_image = tf.reshape(patched_image,(patch_size*l, patch_size*l, 3))
return patched_image
value_col = ['holiday_flg','min_visitors','mean_visitors','median_visitors','max_visitors','count_observations',
'rs1_x','rv1_x','rs2_x','rv2_x','rs1_y','rv1_y','rs2_y','rv2_y','total_reserv_sum','total_reserv_mean',
'total_reserv_dt_diff_mean','date_int','var_max_lat','var_max_long','lon_plus_lat']
nn_col = value_col + ['dow', 'year', 'month', 'air_store_id2', 'air_area_name', 'air_genre_name',
'air_area_name0', 'air_area_name1', 'air_area_name2', 'air_area_name3', 'air_area_name4',
'air_area_name5', 'air_area_name6', 'air_genre_name0', 'air_genre_name1',
'air_genre_name2', 'air_genre_name3', 'air_genre_name4']
X = train.copy()
X_test = test[nn_col].copy()
value_scaler = preprocessing.MinMaxScaler()
for vcol in value_col:
X[vcol] = value_scaler.fit_transform(X[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_test[vcol] = value_scaler.transform(X_test[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_train = list(X[nn_col].T.as_matrix())
 Y_train = X['visitors'].values
nn_train = [X_train, Y_train]
nn_test = [list(X_test[nn_col].T.as_matrix())]
print("Train and test data prepared for NN" )<choose_model_class>
auto = tf.data.experimental.AUTOTUNE
def get_nn_complete_model(train, hidden1_neurons=35, hidden2_neurons=15):

 K.clear_session()
air_store_id = Input(shape=(1,), dtype='int32', name='air_store_id')
 air_store_id_emb = Embedding(len(train['air_store_id2'].unique())+ 1, 15, input_shape=(1,),
name='air_store_id_emb' )(air_store_id)
 air_store_id_emb = keras.layers.Flatten(name='air_store_id_emb_flatten' )(air_store_id_emb)
 dow = Input(shape=(1,), dtype='int32', name='dow')
 dow_emb = Embedding(8, 3, input_shape=(1,), name='dow_emb' )(dow)
 dow_emb = keras.layers.Flatten(name='dow_emb_flatten' )(dow_emb)
 month = Input(shape=(1,), dtype='int32', name='month')
 month_emb = Embedding(13, 3, input_shape=(1,), name='month_emb' )(month)
 month_emb = keras.layers.Flatten(name='month_emb_flatten' )(month_emb)
 air_area_name, air_genre_name = [], []
air_area_name_emb, air_genre_name_emb = [], []
for i in range(7):
area_name_col = 'air_area_name' + str(i)
 air_area_name.append(Input(shape=(1,), dtype='int32', name=area_name_col))
tmp = Embedding(len(train[area_name_col].unique()), 3, input_shape=(1,),
name=area_name_col + '_emb' )(air_area_name[-1])
 tmp = keras.layers.Flatten(name=area_name_col + '_emb_flatten' )(tmp)
 air_area_name_emb.append(tmp)
 if i > 4:
continue
area_genre_col = 'air_genre_name' + str(i)
 air_genre_name.append(Input(shape=(1,), dtype='int32', name=area_genre_col))
tmp = Embedding(len(train[area_genre_col].unique()), 3, input_shape=(1,),
name=area_genre_col + '_emb' )(air_genre_name[-1])
 tmp = keras.layers.Flatten(name=area_genre_col + '_emb_flatten' )(tmp)
 air_genre_name_emb.append(tmp)
 air_genre_name_emb = keras.layers.concatenate(air_genre_name_emb)
 air_genre_name_emb = Dense(4, activation='sigmoid', name='final_air_genre_emb' )(air_genre_name_emb)
 air_area_name_emb = keras.layers.concatenate(air_area_name_emb)
 air_area_name_emb = Dense(4, activation='sigmoid', name='final_air_area_emb' )(air_area_name_emb)
 air_area_code = Input(shape=(1,), dtype='int32', name='air_area_code')
 air_area_code_emb = Embedding(len(train['air_area_name'].unique()), 8, input_shape=(1,), name='air_area_code_emb' )(air_area_code)
 air_area_code_emb = keras.layers.Flatten(name='air_area_code_emb_flatten' )(air_area_code_emb)
 air_genre_code = Input(shape=(1,), dtype='int32', name='air_genre_code')
 air_genre_code_emb = Embedding(len(train['air_genre_name'].unique()), 5, input_shape=(1,),
name='air_genre_code_emb' )(air_genre_code)
 air_genre_code_emb = keras.layers.Flatten(name='air_genre_code_emb_flatten' )(air_genre_code_emb)
 holiday_flg = Input(shape=(1,), dtype='float32', name='holiday_flg')
 year = Input(shape=(1,), dtype='float32', name='year')
 min_visitors = Input(shape=(1,), dtype='float32', name='min_visitors')
 mean_visitors = Input(shape=(1,), dtype='float32', name='mean_visitors')
 median_visitors = Input(shape=(1,), dtype='float32', name='median_visitors')
 max_visitors = Input(shape=(1,), dtype='float32', name='max_visitors')
 count_observations = Input(shape=(1,), dtype='float32', name='count_observations')
 rs1_x = Input(shape=(1,), dtype='float32', name='rs1_x')
 rv1_x = Input(shape=(1,), dtype='float32', name='rv1_x')
 rs2_x = Input(shape=(1,), dtype='float32', name='rs2_x')
 rv2_x = Input(shape=(1,), dtype='float32', name='rv2_x')
 rs1_y = Input(shape=(1,), dtype='float32', name='rs1_y')
 rv1_y = Input(shape=(1,), dtype='float32', name='rv1_y')
 rs2_y = Input(shape=(1,), dtype='float32', name='rs2_y')
 rv2_y = Input(shape=(1,), dtype='float32', name='rv2_y')
 total_reserv_sum = Input(shape=(1,), dtype='float32', name='total_reserv_sum')
 total_reserv_mean = Input(shape=(1,), dtype='float32', name='total_reserv_mean')
 total_reserv_dt_diff_mean = Input(shape=(1,), dtype='float32', name='total_reserv_dt_diff_mean')
 date_int = Input(shape=(1,), dtype='float32', name='date_int')
 var_max_lat = Input(shape=(1,), dtype='float32', name='var_max_lat')
 var_max_long = Input(shape=(1,), dtype='float32', name='var_max_long')
 lon_plus_lat = Input(shape=(1,), dtype='float32', name='lon_plus_lat')
 date_emb = keras.layers.concatenate([dow_emb, month_emb, year, holiday_flg])
 date_emb = Dense(5, activation='sigmoid', name='date_merged_emb' )(date_emb)
 cat_layer = keras.layers.concatenate([holiday_flg, min_visitors, mean_visitors,
median_visitors, max_visitors, count_observations, rs1_x, rv1_x,
rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y,
total_reserv_sum, total_reserv_mean, total_reserv_dt_diff_mean,
date_int, var_max_lat, var_max_long, lon_plus_lat,
date_emb, air_area_name_emb, air_genre_name_emb,
air_area_code_emb, air_genre_code_emb, air_store_id_emb])
 m = Dense(hidden1_neurons, name='hidden1',
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.05, seed=None))(cat_layer)
 m = keras.layers.LeakyReLU(alpha=0.2 )(m)
 m = keras.layers.BatchNormalization()(m)
 m1 = Dense(hidden2_neurons, name='hidden2' )(m)
 m1 = keras.layers.LeakyReLU(alpha=0.2 )(m1)
 m = Dense(1, activation='relu' )(m1)
 inp_ten = [
holiday_flg, min_visitors, mean_visitors, median_visitors, max_visitors, count_observations,
rs1_x, rv1_x, rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y, total_reserv_sum, total_reserv_mean,
total_reserv_dt_diff_mean, date_int, var_max_lat, var_max_long, lon_plus_lat,
dow, year, month, air_store_id, air_area_code, air_genre_code
]
inp_ten += air_area_name
inp_ten += air_genre_name
model = Model(inp_ten, m)
 model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])
 return model<choose_model_class>
import keras.applications
model1 = ensemble.GradientBoostingRegressor(learning_rate=0.2, random_state=3,
n_estimators=180, subsample=0.78, max_depth=10)
 model3 = XGBRegressor(learning_rate=0.2, random_state=3, n_estimators=330, subsample=0.8,
colsample_bytree=0.8, max_depth=10)
 model4 = get_nn_complete_model(train, hidden1_neurons=45)
 model1.fit(train[col], train['visitors'].values)
 print("Model1 trained")
 model3.fit(train[col], train['visitors'].values)
 print("Model3 trained")
 for i in range(5):
model4.fit(nn_train[0], nn_train[1], epochs=8, verbose=0, batch_size=256, shuffle=True)
 model4.fit(nn_train[0], nn_train[1], epochs=3, verbose=0,
batch_size=256, shuffle=True, validation_split=0.15)
 model4.fit(nn_train[0], nn_train[1], epochs=4, verbose=0, batch_size=512, shuffle=True)
 print("Model4 trained")
 preds1 = model1.predict(train[col])
 preds3 = model3.predict(train[col])
 preds4 = pd.Series(model4.predict(nn_train[0] ).reshape(-1)).clip(0, 6.8 ).values
actual_output = train['visitors'].values
print('GradientBoostingRegressor:')
 plot_actual_predicted(actual_output, preds1)
 print('XGBRegressor:')
 plot_actual_predicted(actual_output, preds3)
 print('NeuralNetwork:')
 plot_actual_predicted(actual_output, preds4)
 preds1 = model1.predict(test[col])
 preds3 = model3.predict(test[col])
 preds4 = pd.Series(model4.predict(nn_test[0] ).reshape(-1)).clip(0, 6.8 ).values
test['visitors'] = 0.2*preds1+0.4*preds3+0.4*preds4
test['visitors'] = np.expm1(test['visitors'] ).clip(lower=0.)
sub1 = test[['id','visitors']].copy()
sub1['preds1'] = pd.Series(preds1)
 sub1['preds3'] = pd.Series(preds3)
 sub1['preds4'] = pd.Series(preds4)
 print("Model predictions done.")
<feature_engineering>
import tensorflow as tf
from tensorflow import keras
import keras.backend as k
from tensorflow.keras.layers import Conv2D, Dense, Dropout, GlobalAveragePooling2D, BatchNormalization
from tensorflow.keras import Sequential
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
dfs = { re.search('/([^/\.]*)\.csv', fn ).group(1):
pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
for k, v in dfs.items() : locals() [k] = v
wkend_holidays = date_info.apply(
(lambda x:(x.day_of_week=='Sunday' or x.day_of_week=='Saturday')and x.holiday_flg==1), axis=1)
 date_info.loc[wkend_holidays, 'holiday_flg'] = 0
date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5
visit_data = air_visit_data.merge(date_info, left_on='visit_date', right_on='calendar_date', how='left')
 visit_data.drop('calendar_date', axis=1, inplace=True)
 visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
 wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).reset_index()
visitors.rename(columns={0:'visitors'}, inplace=True)
 sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=[
'air_store_id', 'day_of_week', 'holiday_flg'], how='left')
 missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[visitors.holiday_flg==0], on=('air_store_id', 'day_of_week'),
how='left')['visitors_y'].values
sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sub2 = sample_submission[['id', 'visitors']].copy()
sub2 = sub2.fillna(-1 )<save_to_csv>
model = keras.models.load_model('.. /input/finall/effnetb0model9.h5', compile = False,custom_objects = {
'FixedDropout':FixedDropout} )
<load_from_csv><EOS> 
if os.path.exists(TEST_PATH):
submission_csv = pd.read_csv(test_csv_path)
 test_image_paths = [TEST_PATH + '/' + x + '.tiff' for x in submission_csv['image_id'].values.tolist() ]
test_dat =(
 tf.data.Dataset.from_tensor_slices(test_image_paths)
.map(
 lambda filename: tf.py_function(
 func = read_image,
inp = [filename],
Tout = [tf.uint8]),
num_parallel_calls = auto)
.map(
 lambda image: tf.py_function(
 func = compute_coords,
inp = [image],
Tout= [tf.uint8, tf.int32]),
num_parallel_calls = auto)
.map(
 patch_image,
num_parallel_calls = auto)
.batch(BATCH_SIZE)
.prefetch(auto)
)
 preds = model.predict(test_dat)
 preds = np.sum(((preds > 0.5)*1), axis = 1)
 submission_csv['isup_grade'] = preds
submission_csv['isup_grade'] = submission_csv['isup_grade'].astype(int)
 submission_csv.to_csv('submission.csv', index = False)
 else:
submission_csv = pd.read_csv(test_csv_path)
 submission_csv.to_csv('submission.csv', index = False )