Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
conv = Conv2D(filters=32, kernel_size=3, strides=1,
padding="SAME" )
all_data['item_cnt_month_lag_mean'] = all_data[['item_cnt_month_lag1',
'item_cnt_month_lag2',
'item_cnt_month_lag3']].mean(axis=1 )<feature_engineering>
max_pool = MaxPool2D(pool_size=2 )
all_data[lag_features_to_clip + ['item_cnt_month', 'item_cnt_month_lag_mean']] = all_data[lag_features_to_clip +['item_cnt_month', 'item_cnt_month_lag_mean']].clip(0, 20 )<feature_engineering>
global_avg_pool = GlobalAvgPool2D()
all_data['lag_grad1'] = all_data['item_cnt_month_lag1']/all_data['item_cnt_month_lag2']
all_data['lag_grad1'] = all_data['lag_grad1'].replace([np.inf, -np.inf],
np.nan ).fillna(0)
 all_data['lag_grad2'] = all_data['item_cnt_month_lag2']/all_data['item_cnt_month_lag3']
all_data['lag_grad2'] = all_data['lag_grad2'].replace([np.inf, -np.inf],
np.nan ).fillna(0 )<feature_engineering>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.callbacks import ReduceLROnPlateau
from functools import partial
all_data['brand_new'] = all_data['first_sale_date'] == all_data['date_block_num']<drop_column>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 print(train.shape)
 print(test.shape )
all_data['duration_after_first_sale'] = all_data['date_block_num'] - all_data['first_sale_date']
all_data = all_data.drop('first_sale_date', axis=1 )<feature_engineering>
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_test = test
Y_train.value_counts()
all_data['month'] = all_data['date_block_num']%12<prepare_x_and_y>
X_train = X_train / 255.0
X_test = X_test / 255.0
X_train = all_data[all_data['date_block_num'] < 33]
X_train = X_train.drop(['item_cnt_month'], axis=1)
 X_valid = all_data[all_data['date_block_num'] == 33]
X_valid = X_valid.drop(['item_cnt_month'], axis=1)
 X_test = all_data[all_data['date_block_num'] == 34]
X_test = X_test.drop(['item_cnt_month'], axis=1)
 y_train = all_data[all_data['date_block_num'] < 33]['item_cnt_month']
y_valid = all_data[all_data['date_block_num'] == 33]['item_cnt_month']
del all_data
gc.collect() ;<create_dataframe>
print(Y_train[0:5])
 Y_train = to_categorical(Y_train, num_classes = 10)
 print(Y_train[0:5] )
params = {'metric': 'rmse',
'num_leaves': 255,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'force_col_wise' : True,
'random_state': 10}
cat_features = ['shop_id', 'city', 'item_category_id', 'category', 'month']
dtrain = lgb.Dataset(X_train, y_train)
 dvalid = lgb.Dataset(X_valid, y_valid)
 lgb_model = lgb.train(params=params,
train_set=dtrain,
num_boost_round=1500,
valid_sets=(dtrain, dvalid),
early_stopping_rounds=150,
categorical_feature=cat_features,
verbose_eval=100 )<save_to_csv>
random_seed = 2
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.15, random_state=random_seed )
preds = lgb_model.predict(X_test ).clip(0,20)
 submission['item_cnt_month'] = preds
submission.to_csv('submission.csv', index=False )<drop_column>
DefaultConv2D = partial(Conv2D, kernel_size=3, activation='relu', padding="SAME")
 model = Sequential([
DefaultConv2D(filters=32, kernel_size=5, input_shape=[28, 28, 1]),
DefaultConv2D(filters=32, kernel_size=5),
MaxPooling2D(pool_size=2),
DefaultConv2D(filters=64),
DefaultConv2D(filters=64),
MaxPooling2D(pool_size=2),
Flatten() ,
Dense(units=256, activation='relu'),
Dropout(0.5),
Dense(units=10, activation='softmax'),
] )
del X_train, y_train, X_valid, y_valid, X_test, lgb_model, dtrain, dvalid
gc.collect() ;<load_from_csv>
model.compile(loss="categorical_crossentropy", optimizer="nadam", metrics=["accuracy"] )
sales_train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 test_ids = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 sales_train = sales_train.drop(['date'], axis=1)
 sales_train.head()<merge>
reduce_learning_rate = ReduceLROnPlateau(monitor = 'val_acc', patience = 3, verbose = 1, factor = 0.3, min_lr = 0.00001 )
ids = list(np.arange(test_ids['ID'].max() +1)) *(sales_train['date_block_num'].max() +1)
 dates = list(np.arange(sales_train['date_block_num'].max() +1)) *(test_ids['ID'].max() +1)
 dates.sort()
date_id_dict = {'ID' : ids, 'date_block_num' : dates}
date_id_df = pd.DataFrame.from_dict(date_id_dict)
 date_id_df = date_id_df.merge(test_ids, on='ID')
 date_id_df.head()<merge>
history = model.fit(X_train, Y_train, batch_size = 100, epochs = 20,validation_data =(X_val,Y_val), callbacks=[reduce_learning_rate] )
grouped = sales_train.groupby(['date_block_num', 'shop_id', 'item_id'], as_index=False)
 item_cnt = pd.DataFrame(grouped.sum())
 item_cnt = item_cnt.drop(['item_price'], axis=1)
 grouped = sales_train.groupby(['shop_id', 'item_id'])
 avg_price = pd.DataFrame(grouped.mean() ['item_price'])
 monthly_sales = item_cnt.merge(avg_price, on=['shop_id', 'item_id'])
 monthly_sales = monthly_sales.merge(test_ids, on=['shop_id', 'item_id'])
 monthly_sales.head()<merge>
score = model.evaluate(X_val, Y_val, verbose=0)
 print('Validation loss:', score[0])
 print('Validation accuracy:', score[1] )
item_price = monthly_sales[['item_price', 'ID']]
monthly_sales = monthly_sales.drop(['item_price'], axis=1)
 monthly_sales = date_id_df.merge(monthly_sales, how='left', on=['ID', 'date_block_num'])
 monthly_sales = monthly_sales.drop(['shop_id_y', 'item_id_y'], axis=1)
 monthly_sales['item_cnt_day'].fillna(0, inplace=True)
 monthly_sales = monthly_sales.merge(item_price, how='left', on='ID')
 monthly_sales['item_price'].fillna(monthly_sales['item_price'].mean() , inplace=True)
 monthly_sales = monthly_sales.drop_duplicates()
column_dict = {'shop_id_x' : 'shop_id', 'item_id_x' : 'item_id', 'item_cnt_day' : 'item_cnt_month', 'item_price' : 'avg_price'}
monthly_sales = monthly_sales.rename(columns=column_dict)
 monthly_sales.head()<merge>
print("[INFO] evaluating network...")
 predictions = model.predict(X_val)
 print(classification_report(Y_val.argmax(axis=1),predictions.argmax(axis=1)) )
monthly_sales = monthly_sales.merge(items, on='item_id')
 monthly_sales = monthly_sales.drop(['item_name'], axis=1)
 month = pd.DataFrame([x%12+1 for x in monthly_sales['date_block_num']], columns=['month'])
 year = pd.DataFrame([np.floor(x/12)+2013 for x in monthly_sales['date_block_num']], columns=['year'])
 monthly_sales = pd.concat([monthly_sales, month, year], axis=1)
 monthly_sales = monthly_sales[['ID', 'date_block_num', 'shop_id', 'item_category_id', 'avg_price', 'month', 'year', 'item_cnt_month']]
monthly_sales.head()<merge>
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
def calculate_item_cnt_lagged(df, lag):
tmp = df[['date_block_num', 'ID', 'item_cnt_month']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'ID', 'item_cnt_lag'+str(lag)]
shifted.date_block_num = shifted.date_block_num + lag
df = pd.merge(df, shifted, on=['date_block_num', 'ID'], how='left')
 return df
for lag in range(1, 13):
monthly_sales = calculate_item_cnt_lagged(monthly_sales, lag)
 monthly_sales = monthly_sales[monthly_sales['date_block_num'] > 11]
monthly_sales.head()<merge>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist.csv",index=False )