Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
%load_ext autoreload
%autoreload 2
np.random.seed(1)
 w.filterwarnings('ignore' )
gc.collect()
df = downcast(df )<groupby>
train_file = pd.read_csv(os.path.join(main_path, "train.csv"))
test_file = pd.read_csv(os.path.join(main_path, "test.csv"))
def matching_name_cat_age(df,n,all_shops):
temp_cols = [f'same_name{n}catage_cnt','date_block_num', f'item_name_first{n}','item_age','category_id']
if all_shops:
temp_cols[0] += '_all_shops'
else:
temp_cols += ['shop_id']
temp = []
for i in range(1,35):
block =(
 df 
.query(f'date_block_num < {i}')
.groupby(temp_cols[2:])
.agg({'item_cnt':'mean'})
.reset_index() 
.rename(columns={'item_cnt':temp_cols[0]})
)
 block = block[block[temp_cols[0]]>0]
block['date_block_num'] = i
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=temp_cols[1:], how='left')
 return df
for n in [4,6,11]:
for all_shops in [True,False]:
df = matching_name_cat_age(df,n,all_shops )<data_type_conversions>
print("Training file : ")
 train_file.head(3 ).iloc[:,:17]
df = downcast(df)
 int8_cols = [
'item_cnt','month','group_id','shop_type',
'shop_city','shop_id','date_block_num','category_id',
'item_age',
]
int16_cols = [
'item_id','item_name_first4',
'item_name_first6','item_name_first11'
]
for col in int8_cols:
df[col] = df[col].astype('int8')
 for col in int16_cols:
df[col] = df[col].astype('int16' )<merge>
print("Testing file : ")
 test_file.head(3 ).iloc[:,:17]
def nearby_item_data(df,col):
if col in ['item_cnt_unclipped_lag1','item_cnt_lag1to12']:
cols = ['date_block_num', 'shop_id', 'item_id']
temp = df[cols + [col]]
else:
cols = ['date_block_num', 'item_id']
temp = df.groupby(cols ).agg({col:'first'} ).reset_index() [cols + [col]]
temp.columns = cols + [f'below_{col}']
temp['item_id'] += 1
df = pd.merge(df, temp, on=cols, how='left')
 temp.columns = cols + [f'above_{col}']
temp['item_id'] -= 2
df = pd.merge(df, temp, on=cols, how='left')
 return df
item_cols = ['item_cnt_unclipped_lag1','item_cnt_lag1to12',
'item_cnt_all_shops_lag1','item_cnt_all_shops_lag1to12']
for col in item_cols:
df = nearby_item_data(df,col)
 del temp<feature_engineering>
train_file_norm = train_file.iloc[:, 1:] / 255.0
test_file_norm = test_file / 255.0
results = Counter()
items['item_name'].str.split().apply(results.update)
 words = []
cnts = []
for key, value in results.items() :
words.append(key)
 cnts.append(value)
 counts = pd.DataFrame({'word':words,'count':cnts})
 common_words = counts.query('count>200' ).word.to_list()
for word in common_words:
items[f'{word}_in_name'] = items['item_name'].str.contains(word ).astype('int8')
 drop_cols = [
'item_id','category_id','item_name','item_name_first4',
'item_name_first6','item_name_first11',
'category_name','group_name','group_id'
]
items = items.drop(columns=drop_cols )<merge>
num_examples_train = train_file.shape[0]
num_examples_test = test_file.shape[0]
n_h = 32
n_w = 32
n_c = 3
df = df.join(items, on='item_id' )<categorify>
Train_input_images = np.zeros(( num_examples_train, n_h, n_w, n_c))
Test_input_images = np.zeros(( num_examples_test, n_h, n_w, n_c))
def binary_encode(df, letters, cols):
encoder = ce.BinaryEncoder(cols=[f'item_name_first{letters}'], return_df=True)
 temp = encoder.fit_transform(df[f'item_name_first{letters}'])
 df = pd.concat([df,temp], axis=1)
 del df[f'item_name_first{letters}_0']
name_cols = [f'item_name_first{letters}_{x}' for x in range(1,cols)]
df[name_cols] = df[name_cols].astype('int8')
 return df
df = binary_encode(df, 11, 15)
 del df['item_name_first4'], df['item_name_first6']<categorify>
for example in range(num_examples_train):
Train_input_images[example,:28,:28,0] = train_file.iloc[example, 1:].values.reshape(28,28)
 Train_input_images[example,:28,:28,1] = train_file.iloc[example, 1:].values.reshape(28,28)
 Train_input_images[example,:28,:28,2] = train_file.iloc[example, 1:].values.reshape(28,28)
 for example in range(num_examples_test):
Test_input_images[example,:28,:28,0] = test_file.iloc[example, :].values.reshape(28,28)
 Test_input_images[example,:28,:28,1] = test_file.iloc[example, :].values.reshape(28,28)
 Test_input_images[example,:28,:28,2] = test_file.iloc[example, :].values.reshape(28,28 )
df.to_pickle('df_complete.pkl' )<set_options>
for example in range(num_examples_train):
Train_input_images[example] = cv2.resize(Train_input_images[example],(n_h, n_w))
for example in range(num_examples_test):
Test_input_images[example] = cv2.resize(Test_input_images[example],(n_h, n_w))
%reset -f<set_options>
Train_labels = np.array(train_file.iloc[:, 0] )
pd.set_option('display.max_rows', 160)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 30)
 warnings.filterwarnings("ignore" )<prepare_x_and_y>
train_datagen = ImageDataGenerator(
 rotation_range=27,
width_shift_range=0.3,
height_shift_range=0.2,
shear_range=0.3,
zoom_range=0.2,
horizontal_flip=False)
 validation_datagen = ImageDataGenerator()
df = pd.read_pickle('.. /input/files-top-scoring-notebook-output-exploration/df_complete.pkl')
 X_train = df[~df.date_block_num.isin([0,1,33,34])]
y_train = X_train['item_cnt']
del X_train['item_cnt']
X_val = df[df['date_block_num']==33]
y_val = X_val['item_cnt']
del X_val['item_cnt']
X_test = df[df['date_block_num']==34].drop(columns='item_cnt')
 X_test = X_test.reset_index()
del X_test['index']
del df<train_model>
pretrained_model = keras.applications.resnet50.ResNet50(input_shape=(n_h, n_w, n_c),
include_top=False, weights='imagenet')
 model = keras.Sequential([
pretrained_model,
keras.layers.Flatten() ,
keras.layers.Dense(units=60, activation='relu'),
keras.layers.Dense(units=10, activation='softmax')
 ] )
def build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50,
categorical_feature=cat_features)
 return model<train_model>
Optimizer = 'RMSprop'
model.compile(optimizer=Optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] )
params = {
'objective': 'rmse',
'metric': 'rmse',
'num_leaves': 1023,
'min_data_in_leaf':10,
'feature_fraction':0.7,
'learning_rate': 0.01,
'num_rounds': 1000,
'early_stopping_rounds': 30,
'seed': 1
}
cat_features = ['category_id','month','shop_id','shop_city']
lgb_model = build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features)
 lgb_model.save_model('initial_lgb_model.txt' )<save_to_csv>
train_images, dev_images, train_labels, dev_labels = train_test_split(Train_input_images,
Train_labels,
test_size=0.1, train_size=0.9,
shuffle=True,
random_state=44)
 test_images = Test_input_images
submission = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
 submission['item_cnt_month'] = lgb_model.predict(X_test ).clip(0,20)
 submission[['ID', 'item_cnt_month']].to_csv('initial_lgb_submission.csv', index=False )<load_from_csv>
class myCallback(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')> 0.999999):
print("Stop training!")
 self.model.stop_training = True
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
for i in [r'[^\w\d\s]', r'\bthe\b', r'\bin\b', r'\bfor\b', r'\bof\b', r'\bd\b', r'\bis\b', r'\bon\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(' ', '')
 items = items.join(categories.set_index('category_id'), on='category_id')
 shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'<predict_on_test>
EPOCHS = 5
batch_size = 212
history = model.fit_generator(train_datagen.flow(train_images,train_labels, batch_size=batch_size),
steps_per_epoch=train_images.shape[0] / batch_size,
epochs=EPOCHS,
validation_data=validation_datagen.flow(dev_images,dev_labels,
batch_size=batch_size),
validation_steps=dev_images.shape[0] / batch_size,
callbacks=[callbacks] )
X_train['lgb_pred'] = lgb_model.predict(X_train ).clip(0,20)
 X_train['target'] = y_train
X_train['sq_err'] =(X_train['lgb_pred']-X_train['target'])**2
X_val['lgb_pred'] = lgb_model.predict(X_val ).clip(0,20)
 X_val['target'] = y_val
X_val['sq_err'] =(X_val['lgb_pred']-X_val['target'])**2
X_test['lgb_pred'] = lgb_model.predict(X_test ).clip(0,20 )<feature_engineering>
submission = pd.read_csv('.. /input/digit-recognizer-submission/submission.csv')
 submission.to_csv('digit_submission.csv', index=False )