Kaggle competition: Классификация изображений
<SOS> metric: 
meanfscorebeta Kaggle data source: img-minor-2018<merge>
warnings.filterwarnings('ignore' )
%%time
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True )<merge>
def getTrain(root_dir='train'):
samples = []
labels = []
names = []
for class_dir in ['1_�⮫�', '4_�����', '2_��\u1ae0', '3_������']:
full_path = '{}/{}'.format(root_dir, class_dir)
 curr_class = int(class_dir[0])- 1
for image_name in os.listdir(full_path):
img = Image.open('{}/{}'.format(full_path, image_name))
img.load()
samples.append(img)
 labels.append(curr_class)
 names.append(image_name)
 random_shuffle = np.random.choice(len(samples), len(samples), replace=False)
 samples = np.array(samples, dtype='object')[random_shuffle]
labels = np.array(labels)[random_shuffle]
names = np.array(names)[random_shuffle]
return samples, labels, names
def getTest(root_dir='test'):
samples = []
names = []
for image_name in os.listdir(root_dir):
img = Image.open('{}/{}'.format(root_dir, image_name))
img.load()
samples.append(img)
 names.append(image_name)
 samples = np.array(samples, dtype='object')
 names = np.array(names)
 return samples, names
%%time
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True )<merge>
X_train, y_train, names_train = getTrain('.. /input/img_kagl_train')
 print('Train loaded!')
 X_test, names_test = getTest(root_dir='.. /input/img_kagl_test/test')
 print('Test loaded!' )
%%time
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True )<merge>
train_data, test_data, train_labels, test_labels = train_test_split(X_train, y_train, test_size=0.2 )
%%time
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True )<merge>
print('tensoring train')
 batch_size = 64
tensor_x = torch.stack([torch.Tensor(transformer(i)) for i in train_data])
 tensor_y = torch.from_numpy(np.array(train_labels))
tensor_train = torch.utils.data.TensorDataset(tensor_x,tensor_y)
 train_load = torch.utils.data.DataLoader(dataset = tensor_train,
batch_size = batch_size,
shuffle = True)
 print('tensoring test')
 tensor_x = torch.stack([torch.Tensor(transformer(i)) for i in test_data])
 tensor_y = torch.from_numpy(np.array(test_labels))
tensor_test = torch.utils.data.TensorDataset(tensor_x, tensor_y)
 test_load = torch.utils.data.DataLoader(dataset = tensor_test,
batch_size = batch_size,
shuffle = False)
 print('tensoring pred')
 tensor_x = torch.stack([torch.Tensor(transformer(i)) for i in X_test])
 tensor_pred = torch.utils.data.TensorDataset(tensor_x)
 pred_load = torch.utils.data.DataLoader(dataset = tensor_pred,
batch_size = batch_size,
shuffle = False
)
%%time
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True )<merge>
class CNN(nn.Module):
def __init__(self):
super(CNN,self ).__init__()
self.cnn1 = nn.Conv2d(in_channels=3, out_channels=8, kernel_size=3, stride=1, padding=1)
 self.batchnorm1 = nn.BatchNorm2d(8)
 self.relu = nn.ReLU()
self.maxpool1 = nn.MaxPool2d(kernel_size=2)
 self.cnn2 = nn.Conv2d(in_channels=8, out_channels=32, kernel_size=5, stride=1, padding=2)
 self.batchnorm2 = nn.BatchNorm2d(32)
 self.maxpool2 = nn.MaxPool2d(kernel_size=2)
 self.fc1 = nn.Linear(in_features=2048, out_features=1000)
 self.dropout = nn.Dropout(p=0.5)
 self.fc2 = nn.Linear(in_features=1000, out_features=4)
 def forward(self, X):
out = self.cnn1(X)
 out = self.batchnorm1(out)
 out = self.relu(out)
 out = self.maxpool1(out)
 out = self.cnn2(out)
 out = self.batchnorm2(out)
 out = self.relu(out)
 out = self.maxpool2(out)
 out = out.view(-1, 2048)
 out = self.fc1(out)
 out = self.relu(out)
 out = self.dropout(out)
 out = self.fc2(out)
 return out
%%time
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True )<merge>
def calculate_loss_accuracy(load, model, loss_fn):
total = 0
correct = 0
accuracy = 0
loss = 0
for images, labels in load:
outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 loss += loss_fn(outputs, labels ).item()
total += labels.size(0)
 correct +=(predicted.cpu() == labels.cpu() ).sum()
accuracy = float(correct)/ total * 100
loss /= len(load.dataset)
 return loss, accuracy
%%time
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True )<merge>
model = CNN()
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , 0.01)
 train_loss_history = []
test_loss_history = []
train_accuracy_history = []
test_accuracy_history = []
iter = 0
for epoch in range(40):
for i,(images, labels)in enumerate(train_load):
iter += 1
optimizer.zero_grad()
outputs = model(images)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
train_loss, train_accuracy = calculate_loss_accuracy(train_load, model, loss_fn)
 train_loss_history.append(train_loss)
 train_accuracy_history.append(train_accuracy)
 test_loss, test_accuracy = calculate_loss_accuracy(test_load, model, loss_fn)
 test_loss_history.append(test_loss)
 test_accuracy_history.append(test_accuracy)
 print(epoch, 'Done! Train Loss: {}, Test Loss: {}, Train Accuracy {:.2f}%, Test Accuracy {:.2f}%'.format(train_loss, test_loss, train_accuracy, test_accuracy))
%%time
group = train_data.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train_data.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True )<merge>
model = CNN()
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , 0.01)
 train_loss_history = []
test_loss_history = []
train_accuracy_history = []
test_accuracy_history = []
iter = 0
for epoch in range(10):
for i,(images, labels)in enumerate(train_load):
iter += 1
optimizer.zero_grad()
outputs = model(images)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
train_loss, train_accuracy = calculate_loss_accuracy(train_load, model, loss_fn)
 train_loss_history.append(train_loss)
 train_accuracy_history.append(train_accuracy)
 test_loss, test_accuracy = calculate_loss_accuracy(test_load, model, loss_fn)
 test_loss_history.append(test_loss)
 test_accuracy_history.append(test_accuracy)
 print(epoch, 'Done! Train Loss: {}, Test Loss: {}, Train Accuracy {:.2f}%, Test Accuracy {:.2f}%'.format(train_loss, test_loss, train_accuracy, test_accuracy))
view_graphs(train_accuracy_history, test_accuracy_history, train_loss_history, test_loss_history, title='' )
%%time
group = train_data.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True )<categorify>
def get_preds(pred_load, model):
preds = []
for images in pred_load:
images = images[0]
images = images
model.training = False
outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 preds.append(predicted)
 ans = []
for i in range(len(preds)) :
for j in range(len(preds[i])) :
ans.append(preds[i][j])
 return np.array(ans )
matrix['month'] = matrix['date_block_num'] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
classes = ['Столы', 'Кресла', 'Диваны', 'Стулья']
preds_train = get_preds(train_load, model ).reshape(-1, 1)
 preds_validation = get_preds(test_load, model ).reshape(-1, 1 )
%%time
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for row in matrix.itertuples() :
idx = getattr(row,'Index')
 item_id = getattr(row,'item_id')
 shop_id = getattr(row,'shop_id')
 date_block_num = getattr(row,'date_block_num')
 item_cnt_month = getattr(row,'item_cnt_month')
 key = str(item_id)+' '+str(shop_id)
 if key not in cache:
if item_cnt_month!=0:
cache[key] = date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = date_block_num - last_date_block_num
cache[key] = date_block_num<data_type_conversions>
ans = get_preds(pred_load, model )
<categorify><EOS> 
answers = pd.DataFrame()
answers['Image'] = names_test
answers['Type'] = ans + 1
answers.to_csv('predict2.csv', index=False )