Kaggle competition: ML Hackathon 2019 Q2
<SOS> metric: categorizationaccuracy Kaggle data source: ml-hackathon-2019-q2<concatenate>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.layers import Activation,Dropout,Dense,Conv2D,AveragePooling2D,Flatten,ZeroPadding2D,MaxPooling2D
from keras import optimizers
from sklearn.model_selection import train_test_split
from keras.models import Sequential
import seaborn as sns
from sklearn.metrics import accuracy_score
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import img_to_array
from keras.preprocessing.image import load_img
import math
import cv2
import imageio
from os import listdir
import warnings
import filecmp
from PIL import Image
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
def rotate(image, angle):
image_center = tuple(np.array(image.shape[1::-1])/ 2)
 rot_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0)
 result = cv2.warpAffine(image, rot_mat, image.shape[1::-1], flags=cv2.INTER_LINEAR)
 return result
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
def reorder(old_list,order):
new_list = []
for i in order:
new_list.append(old_list[i])
 return new_list
np.random.seed(seed=42)
 indices = np.arange(len(fruit_labels))
np.random.shuffle(indices)
 indices = indices.tolist()
fruit_labels = reorder(fruit_labels,indices)
 fruit_images = reorder(fruit_images,indices)
 image_array = np.array(fruit_images)
 label_array = np.array(fruit_labels )
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
label_array=to_categorical(label_array,3 )
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
X_train, X_test, y_train, y_test = train_test_split(image_array,label_array, test_size=0.2)
 X_train, X_val, y_train, y_val=train_test_split(X_train, y_train, test_size=0.3 )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
model = Sequential([
Conv2D(32,(3,3), activation='relu', input_shape=(50,50,3),padding='same'),
MaxPooling2D(2, 2),
Conv2D(64,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Conv2D(128,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Flatten() ,
Dropout(0.50),
Dense(128, activation='relu'),
Dense(3, activation='softmax')
 ])
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
l=[25]
for i in range(len(l)) :
model = Sequential([
Conv2D(32,(3,3), activation='relu', input_shape=(50,50,3),padding='same'),
MaxPooling2D(2, 2),
Conv2D(64,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Conv2D(128,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Flatten() ,
Dropout(0.50),
Dense(128, activation='relu'),
Dense(3, activation='softmax')
 ])
 model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.fit(X_train,y_train,epochs=l[i],verbose=1)
 modelsub=model.predict(X_val,batch_size=None, verbose=1)
 c=0
f=0
for i in range(modelsub.shape[0]):
if np.argmax(modelsub[i])==np.argmax(y_val[i]):
c+=1
else:
f+=1
accuracy=c/(c+f)
 print('Test Accuracy:',accuracy*100)
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
modelsub=model.predict(X_test,batch_size=None, verbose=1)
 c=0
f=0
for i in range(modelsub.shape[0]):
if np.argmax(modelsub[i])==np.argmax(y_test[i]):
c+=1
else:
f+=1
accuracy=c/(c+f)
 print('Test Accuracy:',accuracy*100 )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
sub_df=pd.read_csv('.. /input/sample_submission_q2.csv')
 test_id = sub_df['ID']
test_id=test_id.astype(str)
 test_path='.. /input/test/Test/'
test = get_images(ids=test_id, filepath=test_path)
<merge><EOS> 
l=[]
iid=np.arange(1,113,1)
 c=model.predict(test,batch_size=None, verbose=1)
 for i in range(len(c)) :
l.append(np.argmax(c[i]))
df=pd.DataFrame({'ID':iid,'Category':l
},index=iid)
 print(l)
 df.to_csv("q2.csv",index=False )