Kaggle competition: Expedia Hotel Recommendations
N = train_df.shape[0]
x_train = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(train_df['id_code'])) :
x_train[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png'
)<prepare_x_and_y>
train = pd.read_csv('.. /input/train.csv',
dtype={'is_booking':bool,'srch_destination_id':np.int32, 'hotel_cluster':np.int32},
usecols=['srch_destination_id','is_booking','hotel_cluster'],
chunksize=1000000)
 aggs = []
print('-'*38)
 for chunk in train:
agg = chunk.groupby(['srch_destination_id',
'hotel_cluster'])['is_booking'].agg(['sum','count'])
 agg.reset_index(inplace=True)
 aggs.append(agg)
 print('.',end='')
 print('')
 aggs = pd.concat(aggs, axis=0)
 aggs.head()
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png'
)<categorify>
def most_popular(group, n_max=5):
relevance = group['relevance'].values
hotel_cluster = group['hotel_cluster'].values
most_popular = hotel_cluster[np.argsort(relevance)[::-1]][:n_max]
return np.array_str(most_popular)[1:-1]
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )<categorify>
most_pop = agg.groupby(['srch_destination_id'] ).apply(most_popular)
 most_pop = pd.DataFrame(most_pop ).rename(columns={0:'hotel_cluster'})
 most_pop.head()
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
test = pd.read_csv('.. /input/test.csv',
dtype={'srch_destination_id':np.int32},
usecols=['srch_destination_id'], )
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.15,
random_state=2019
)<train_model>
test = test.merge(most_pop, how='left',left_on='srch_destination_id',right_index=True)
 test.head()
class MixupGenerator() :
def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):
self.X_train = X_train
self.y_train = y_train
self.batch_size = batch_size
self.alpha = alpha
self.shuffle = shuffle
self.sample_num = len(X_train)
 self.datagen = datagen
def __call__(self):
while True:
indexes = self.__get_exploration_order()
itr_num = int(len(indexes)//(self.batch_size * 2))
for i in range(itr_num):
batch_ids = indexes[i * self.batch_size * 2:(i + 1)* self.batch_size * 2]
X, y = self.__data_generation(batch_ids)
 yield X, y
def __get_exploration_order(self):
indexes = np.arange(self.sample_num)
 if self.shuffle:
np.random.shuffle(indexes)
 return indexes
def __data_generation(self, batch_ids):
_, h, w, c = self.X_train.shape
l = np.random.beta(self.alpha, self.alpha, self.batch_size)
 X_l = l.reshape(self.batch_size, 1, 1, 1)
 y_l = l.reshape(self.batch_size, 1)
 X1 = self.X_train[batch_ids[:self.batch_size]]
X2 = self.X_train[batch_ids[self.batch_size:]]
X = X1 * X_l + X2 *(1 - X_l)
 if self.datagen:
for i in range(self.batch_size):
X[i] = self.datagen.random_transform(X[i])
 X[i] = self.datagen.standardize(X[i])
 if isinstance(self.y_train, list):
y = []
for y_train_ in self.y_train:
y1 = y_train_[batch_ids[:self.batch_size]]
y2 = y_train_[batch_ids[self.batch_size:]]
y.append(y1 * y_l + y2 *(1 - y_l))
else:
y1 = self.y_train[batch_ids[:self.batch_size]]
y2 = self.y_train[batch_ids[self.batch_size:]]
y = y1 * y_l + y2 *(1 - y_l)
 return X, y<train_model>
test.hotel_cluster.isnull().sum()
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)
 mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen() )()<define_variables>
most_pop_all = agg.groupby('hotel_cluster')['relevance'].sum().nlargest(5 ).index
most_pop_all = np.array_str(most_pop_all)[1:-1]
most_pop_all
true_labels = np.array([1, 0, 1, 1, 0, 1])
 pred_labels = np.array([1, 0, 0, 0, 0, 1] )<compute_test_metric>
test.hotel_cluster.fillna(most_pop_all,inplace=True )
accuracy_score(true_labels, pred_labels )<compute_test_metric>
test.hotel_cluster.to_csv('predicted_with_pandas.csv',header=True, index_label='id' )