Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
print(tf.__version__)
le = LabelEncoder()
pd.set_option('display.max_rows', 400)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 40)
 warnings.filterwarnings("ignore" )<load_from_csv>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test.head()<load_from_csv>
cut=0
nets = 15
epochs = 45
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 pd.DataFrame(categories.category_name.values.reshape(-1, 4))<categorify>
if cut > 0:
n_train=round(cut*train.shape[0])
 train = train[:n_train]
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
X_test = test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 X_test = X_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10 )
categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
categories['group_id'] = le.fit_transform(categories.group_name.values)
 categories.sample(5 )<load_from_csv>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.15,
height_shift_range=0.15 )
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
items['item_name'] = items['item_name'].str.replace('.', '')
 for i in [r'[^\w\d\s\.]', r'\bthe\b', r'\bin\b', r'\bis\b',
r'\bfor\b', r'\bof\b', r'\bon\b', r'\band\b',
r'\bto\b', r'\bwith\b' , r'\byo\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(r'\b.\b', ' ')
 items['item_name_no_space'] = items['item_name'].str.replace(' ', '')
 items['item_name_first4'] = [x[:4] for x in items['item_name_no_space']]
items['item_name_first6'] = [x[:6] for x in items['item_name_no_space']]
items['item_name_first11'] = [x[:11] for x in items['item_name_no_space']]
del items['item_name_no_space']
items.item_name_first4 = le.fit_transform(items.item_name_first4.values)
 items.item_name_first6 = le.fit_transform(items.item_name_first6.values)
 items.item_name_first11 = le.fit_transform(items.item_name_first11.values)
 items = items.join(categories.set_index('category_id'), on='category_id')
 items.sample(10 )<remove_duplicates>
DEVICE = 'TPU'
if DEVICE == 'TPU':
print('connecting to TPU...')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print('Could not connect to TPU')
 tpu = None
if tpu:
try:
print('Initializing TPU...')
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 distribution_strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('TPU initialized')
 except _:
print('Failed to initialize TPU!')
 else:
DEVICE = 'GPU'
if DEVICE != 'TPU':
print('Using default strategy for CPU and single GPU')
 distribution_strategy = tf.distribute.get_strategy()
if DEVICE == 'GPU':
print('Num GPUs Available: ',
len(tf.config.experimental.list_physical_devices('GPU')))
 print('REPLICAS: ', distribution_strategy.num_replicas_in_sync)
dupes = items[(items.duplicated(subset=['item_name','category_id'],keep=False)) ]
dupes['in_test'] = dupes.item_id.isin(test.item_id.unique())
 dupes = dupes.groupby('item_name' ).agg({'item_id':['first','last'],'in_test':['first','last']})
 dupes = dupes[(dupes[('in_test', 'first')]==False)|(dupes[('in_test', 'last')]==False)]
temp = dupes[dupes[('in_test', 'first')]==True]
keep_first = dict(zip(temp[('item_id', 'last')], temp[('item_id', 'first')]))
temp = dupes[dupes[('in_test', 'first')]==False]
keep_second = dict(zip(temp[('item_id', 'first')], temp[('item_id', 'last')]))
item_map = {**keep_first, **keep_second}<feature_engineering>
with distribution_strategy.scope() :
model = [0] *nets
for j in range(nets):
model[j] = Sequential()
model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Flatten())
 model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 sales =(sales 
.query('0 < item_price < 50000 and 0 < item_cnt_day < 1001')
.replace({
'shop_id':{0:57, 1:58, 11:10},
'item_id':item_map
})
)
 sales = sales[sales['shop_id'].isin(test.shop_id.unique())]
sales['date'] = pd.to_datetime(sales.date,format='%d.%m.%Y')
 sales['weekday'] = sales.date.dt.dayofweek
sales['first_sale_day'] = sales.date.dt.dayofyear
sales['first_sale_day'] += 365 *(sales.date.dt.year-2013)
 sales['first_sale_day'] = sales.groupby('item_id')['first_sale_day'].transform('min' ).astype('int16')
 sales['revenue'] = sales['item_cnt_day']*sales['item_price']<feature_engineering>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
for j in range(nets):
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 j+1,epochs,max(history[j].history['accuracy']),max(history[j].history['val_accuracy'])) )
temp = sales.groupby(['shop_id','weekday'] ).agg({'item_cnt_day':'sum'} ).reset_index()
temp = pd.merge(temp, sales.groupby(['shop_id'] ).agg({'item_cnt_day':'sum'} ).reset_index() , on='shop_id', how='left')
 temp.columns = ['shop_id','weekday', 'shop_day_sales', 'shop_total_sales']
temp['day_quality'] = temp['shop_day_sales']/temp['shop_total_sales']
temp = temp[['shop_id','weekday','day_quality']]
dates = pd.DataFrame(data={'date':pd.date_range(start='2013-01-01',end='2015-11-30')})
 dates['weekday'] = dates.date.dt.dayofweek
dates['month'] = dates.date.dt.month
dates['year'] = dates.date.dt.year - 2013
dates['date_block_num'] = dates['year']*12 + dates['month'] - 1
dates['first_day_of_month'] = dates.date.dt.dayofyear
dates['first_day_of_month'] += 365 * dates['year']
dates = dates.join(temp.set_index('weekday'), on='weekday')
 dates = dates.groupby(['date_block_num','shop_id','month','year'] ).agg({'day_quality':'sum','first_day_of_month':'min'} ).reset_index()
dates.query('shop_id == 28' ).head(15 )<groupby>
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )