Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<categorify>
import os
import math
import numpy as np
import pandas as pd
from itertools import product
from matplotlib import pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
import xgboost as xgb
class SeedlingDataset(Dataset):
def __init__(self, filenames, labels, root_dir, subset=False, transform=None):
self.filenames = filenames
self.labels = labels
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.labels)
 def __getitem__(self, idx):
fullname = join(self.root_dir, self.filenames.iloc[idx])
 image = Image.open(fullname ).convert('RGB')
 if self.transform:
image = self.transform(image)
 return image, self.labels.iloc[idx] 
<import_modules>
sample_submission = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
items = pd.read_csv(os.path.join(data_dir, 'items.csv'))
shops = pd.read_csv(os.path.join(data_dir,'shops.csv'))
sales = pd.read_csv(os.path.join(data_dir, 'sales_train.csv'))
test_data = pd.read_csv(os.path.join(data_dir, 'test.csv'))
from sklearn.model_selection import train_test_split<import_modules>
sales.nunique()
from sklearn.model_selection import train_test_split<prepare_x_and_y>
sales[sales['item_price'] > 100000]
X, y = df.drop(['category_id', 'category'], axis=1), df['category_id']<split>
sales = sales[sales['date_block_num'] > 11]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
 X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=1 )<train_on_grid>
all_data.loc[all_data['shop_id'] == 0, 'shop_id'] = 57
all_data.loc[all_data['shop_id'] == 1, 'shop_id'] = 58
all_data.loc[all_data['shop_id'] == 11, 'shop_id'] = 10
def train(train_loader, model, optimizer, criterion):
model.train()
losses, accuracies = [], []
for batch_idx,(data, target)in(enumerate(train_loader)) :
correct = 0
if use_cuda:
data, target = data.cuda() , target.cuda()
optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 loss.backward()
pred = output.data.max(1)[1]
correct += pred.eq(target.data ).cpu().sum()
accuracy = 100.* correct / len(data)
 optimizer.step()
losses.append(loss.data.item())
 accuracies.append(accuracy)
 return np.mean(losses), np.mean(accuracies)
 def test(test_loader, criterion, model):
test_loss = 0
correct = 0
for data, target in test_loader:
if use_cuda:
data, target = data.cuda() , target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
 output = model(data)
 test_loss += criterion(output, target ).data.item()
pred = output.data.max(1)[1]
correct += pred.eq(target.data ).cpu().sum().item()
test_loss /= len(test_loader)
 accuracy = 100.* correct / len(test_loader.dataset)
 return test_loss, accuracy
def _should_early_stop(recent_validation_losses, validation_loss, early_stopping_rounds):
recent_validation_losses.append(validation_loss)
 if early_stopping_rounds < len(recent_validation_losses):
recent_validation_losses.popleft()
return all(np.diff(recent_validation_losses)> 0)
 return False
def run_train_process(epochs, loaders, model, optimizer, criterion, early_stopping_rounds=10):
epoch_to_results = {}
recent_validation_losses = deque()
for epoch in tqdm(range(epochs)) :
train_loss, train_accuracy = train(loaders['train'], model, optimizer, criterion)
 validation_loss, validation_accuracy = test(loaders['validation'], criterion, model)
 epoch_to_results[epoch] = {
'train_loss': train_loss,
'train_accuracy': train_accuracy,
'validation_loss': validation_loss,
'validation_accuracy': validation_accuracy,
}
if epoch % 5 == 0:
print('[%s] Train Accuracy: %.5f , Validation Accuracy: %.5f' %(epoch, train_accuracy, validation_accuracy))
should_early_stop = _should_early_stop(recent_validation_losses, validation_loss, early_stopping_rounds)
 if should_early_stop:
print(f'Train epoch {epoch} EARLY STOPPING - validation loss has not improved in the last {early_stopping_rounds} rounds')
 break
df_epoch_to_results = pd.DataFrame.from_dict(epoch_to_results ).T
_, test_accuracy = test(loaders['test'], criterion, model)
 return df_epoch_to_results, test_accuracy<import_modules>
lags = [1, 2, 3, 6, 12]
for lag in lags:
lag_col_name = 'target_lag_' + str(lag)
 shifted = all_data[index_cols + ['target']].copy()
shifted.columns = index_cols + [lag_col_name]
shifted['date_block_num'] += lag
all_data = pd.merge(all_data, shifted, on=index_cols, how='left')
 all_data[lag_col_name].fillna(0, inplace=True )
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from tqdm import tqdm, tqdm_notebook<categorify>
item_category_mapping = items[['item_id','item_category_id']].drop_duplicates()
all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id' )
class LeNet(nn.Module):
def __init__(self, num_classes=12, num_rgb=3):
super(LeNet, self ).__init__()
self.conv1 = nn.Conv2d(num_rgb, 6, 5)
 self.conv2 = nn.Conv2d(6, 16, 5)
 self.fc1 = nn.Linear(44944, 120)
 self.fc2 = nn.Linear(120, 84)
 self.fc3 = nn.Linear(84, num_classes)
 def forward(self, x):
out = F.relu(self.conv1(x))
out = F.max_pool2d(out, 2)
 out = F.relu(self.conv2(out))
out = F.max_pool2d(out, 2)
 out = out.view(out.size(0), -1)
 out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
out = out.view(out.size(0), -1)
 out = self.fc3(out)
 return out 
<prepare_x_and_y>
cumsum = all_data.groupby('item_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('item_id')['target'].cumcount()
all_data['item_target_enc_exp'] = cumsum / cumcnt
target_mean = all_data['target'].mean()
all_data['item_target_enc_exp'].fillna(target_mean, inplace=True )
learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 image_size =(224, 224)
 train_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor()
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor()
])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 batch_size = 8
loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment2_result, test_accuracy_experiment2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment2_result<prepare_x_and_y>
cumsum = all_data.groupby('shop_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('shop_id')['target'].cumcount()
all_data['shop_target_enc_exp'] = cumsum / cumcnt
target_mean = all_data['target'].mean()
all_data['shop_target_enc_exp'].fillna(target_mean, inplace=True )
image_size =(224, 224)
 train_set = SeedlingDataset(
 X_train.file, y_train, '.. /input/train/',
transform=transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ]))
means = []
means_sq = []
for img, _ in tqdm_notebook(train_set):
means.append(np.asarray(img, dtype='float32' ).mean(axis=(1,2)))
 means_sq.append(( np.asarray(img, dtype='float32')** 2 ).mean(axis=(1,2)))
 mean_img = np.mean(means, axis=0)
 std_img = np.sqrt(np.mean(means_sq, axis=0)-(mean_img ** 2))<prepare_x_and_y>
cumsum = all_data.groupby('item_category_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('item_category_id')['target'].cumcount()
all_data['item_category_target_enc_exp'] = cumsum / cumcnt
target_mean = all_data['target'].mean()
all_data['item_category_target_enc_exp'].fillna(target_mean, inplace=True )
learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 image_size =(224, 224)
 train_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 batch_size = 8
loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment3_part1_result, test_accuracy_experiment3_part1 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment3_part1_result<train_on_grid>
last_sale_df = []
for d in range(1, 35):
df = sales[sales.date_block_num < d].groupby(['shop_id', 'item_id'], as_index=False)['date_block_num'].max()
df['last_sale_ago'] = d - df.date_block_num
df.date_block_num = d
last_sale_df.append(df)
 last_sale_df = pd.concat(last_sale_df)
 all_data = pd.merge(all_data, last_sale_df, on=['shop_id', 'item_id', 'date_block_num'], how='left')
 all_data['last_sale_ago'].fillna(0, inplace=True )
image_size =(224, 224)
 batch_size = 8
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment3_part2_result, test_accuracy_experiment3_part2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment3_part2_result<choose_model_class>
last_shop_sale_df = []
for d in range(1, 35):
df = sales[sales.date_block_num < d].groupby('shop_id', as_index=False)['date_block_num'].max()
df['last_shop_sale_ago'] = d - df.date_block_num
df.date_block_num = d
last_shop_sale_df.append(df)
 last_shop_sale_df = pd.concat(last_shop_sale_df)
 all_data = pd.merge(all_data, last_shop_sale_df, on=['shop_id', 'date_block_num'], how='left')
 all_data['last_shop_sale_ago'].fillna(0, inplace=True )
class LeNetWithKernelSize3(LeNet):
def __init__(self, num_classes=12, num_rgb=3):
super(LeNetWithKernelSize3, self ).__init__(num_classes, num_rgb)
 self.conv1 = nn.Conv2d(num_rgb, 6, 3)
 self.conv2 = nn.Conv2d(6, 16, 3)
 self.fc1 = nn.Linear(46656, 120)
 self.fc2 = nn.Linear(120, 84)
 self.fc3 = nn.Linear(84, num_classes )<prepare_x_and_y>
last_item_sale_df = []
for d in range(1, 35):
df = sales[sales.date_block_num < d].groupby('item_id', as_index=False)['date_block_num'].max()
df['last_item_sale_ago'] = d - df.date_block_num
df.date_block_num = d
last_item_sale_df.append(df)
 last_item_sale_df = pd.concat(last_item_sale_df)
 all_data = pd.merge(all_data, last_item_sale_df, on=['item_id', 'date_block_num'], how='left')
 all_data['last_item_sale_ago'].fillna(0, inplace=True )
image_size =(224, 224)
 batch_size = 8
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNetWithKernelSize3().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 batch_size = 8
loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment4_part1_result, test_accuracy_experiment4_part1 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment4_part1_result<prepare_x_and_y>
all_data['year_index'] = all_data['date_block_num'] // 12
all_data['month'] = all_data['date_block_num'] % 12 + 1
all_data = all_data.drop(columns='date_block_num' )
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNetWithKernelSize3().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 100
df_experiment4_part2_result, test_accuracy_experiment4_part2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment4_part2_result<feature_engineering>
days = pd.Series([0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31])
 all_data['days_in_month'] = all_data['month'].map(days )
class LeNetWithSigmoidActivation(LeNet):
def forward(self, x):
out = F.sigmoid(self.conv1(x))
out = F.max_pool2d(out, 2)
 out = F.sigmoid(self.conv2(out))
out = F.max_pool2d(out, 2)
 out = out.view(out.size(0), -1)
 out = F.sigmoid(self.fc1(out))
out = F.sigmoid(self.fc2(out))
out = out.view(out.size(0), -1)
 out = self.fc3(out)
 return out<prepare_x_and_y>
year_oh = pd.get_dummies(all_data['year_index'], prefix='year')
 month_oh = pd.get_dummies(all_data['month'], prefix='month')
 all_data_oh = all_data.drop(columns=['shop_id', 'item_id', 'item_category_id', 'year_index', 'month', 'target', 'last_sale_ago', 'last_shop_sale_ago', 'last_item_sale_ago'])
 all_data_oh = pd.concat([all_data_oh, year_oh, month_oh], axis=1)
 all_data_oh.head()
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNetWithSigmoidActivation().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 100
df_experiment5_part1_result, test_accuracy_experiment5_part1 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment5_part1_result<prepare_x_and_y>
train_b_index =(all_data['year_index'] == 2)&(all_data['month'] == 9)
 train_c_index =(all_data['year_index'] == 2)&(all_data['month'] == 10)
 train_a_index = ~train_b_index & ~train_c_index
X_train_a = all_data[train_a_index]
y_train_a = X_train_a['target'].clip(0, 20)
 X_train_a = X_train_a.drop(columns='target')
 X_train_b = all_data[train_b_index]
y_train_b = X_train_b['target'].clip(0, 20)
 X_train_b = X_train_b.drop(columns='target')
 X_train_c = all_data[train_c_index]
y_train_c = X_train_c['target'].clip(0, 20)
 X_train_c = X_train_c.drop(columns='target' )
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters() , lr=learning_rate)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 100
df_experiment5_part2_result, test_accuracy_experiment5_part2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment5_part2_result<choose_model_class>
X_train_a_oh = all_data_oh[train_a_index]
X_train_b_oh = all_data_oh[train_b_index]
X_train_c_oh = all_data_oh[train_c_index]
vgg16 = models.vgg16_bn(pretrained=True)
 print(f'Number of output fearurs of pretrained vgg is: {vgg16.classifier[6].out_features}')
 for param in vgg16.features.parameters() :
param.require_grad = False
num_features = vgg16.classifier[6].in_features
print(f'Last layer of vgg16 has {num_features} input features.')
 print(f'Removing layer: {list(vgg16.classifier.children())[-1]}')
 classifier_layers = list(vgg16.classifier.children())[:-1]
layer_to_add = nn.Linear(num_features, len(np.unique(y)))
 print(f'Adding layer: {layer_to_add}')
 classifier_layers.extend([layer_to_add])
 vgg16.classifier = nn.Sequential(*classifier_layers )<prepare_x_and_y>
model_xgb = xgb.XGBRegressor(max_depth=4, learning_rate=0.5, n_jobs=-1)
 model_xgb.fit(X_train_a, y_train_a )
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
vgg = vgg16.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(vgg.parameters() , lr=0.001, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 30
df_experiment8_result, test_accuracy_experiment8 = run_train_process(epochs, loaders, vgg, optimizer, criterion)
 df_experiment8_result<feature_engineering>
model_mlp = MLPRegressor(hidden_layer_sizes=(100, 100), activation='relu', learning_rate_init=0.01, max_iter=10, shuffle=False, verbose=True)
 model_mlp.fit(X_train_a_oh.values, y_train_a )
df_experiment2_result['experiment'] = 'experiment2'
df_experiment3_part1_result['experiment'] = 'experiment3_part1'
df_experiment3_part2_result['experiment'] = 'experiment3_part2'
df_experiment4_part1_result['experiment'] = 'experiment4_part1'
df_experiment4_part2_result['experiment'] = 'experiment4_part2'
df_experiment5_part1_result['experiment'] = 'experiment5_part1'
df_experiment5_part2_result['experiment'] = 'experiment5_part2'
df_experiment8_result['experiment'] = 'experiment8'
df_all_expirement_results = pd.concat([
df_experiment2_result,
df_experiment3_part1_result,
df_experiment3_part2_result,
df_experiment4_part1_result,
df_experiment4_part2_result,
df_experiment5_part1_result,
df_experiment5_part2_result,
df_experiment8_result
] ).rename_axis('epoch' ).reset_index()
df_all_expirement_results<save_to_csv>
model_rf = RandomForestRegressor(n_estimators=10, criterion='mse', max_depth=None, n_jobs=-1, verbose=1)
 model_rf.fit(X_train_a.values, y_train_a )
df_all_expirement_results.to_csv('df_all_expirement_results.csv', index=False, header=True )<load_from_csv>
y_pred_1 = model_xgb.predict(X_train_b)
 rmse = math.sqrt(mean_squared_error(y_train_b, y_pred_1))
R_score = r2_score(y_train_b, y_pred_1)
 print('XGBoost rmse: ' + str(rmse)+ ', R2: ' + str(R_score))
data_dir = '.. /input/'
sample_submission = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
image_size =(224, 224)
 test_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 kaggle_train_set = SeedlingDataset(
 sample_submission.file,
sample_submission.species,
'.. /input/test/',
transform=test_trans)
 kaggle_test_loader = DataLoader(kaggle_train_set, batch_size=8, shuffle=False, num_workers=0)
 def predict(sample_submission, kaggle_test_loader, model):
predictions = []
for data, _ in kaggle_test_loader:
if use_cuda:
data = data.cuda()
data = Variable(data)
 output = model(data)
 pred = output.data.max(1)[1]
predictions.extend(pred.tolist())
 df_predictions = pd.DataFrame.from_dict({
'file': sample_submission.file,
'species': [num_to_class[p] for p in predictions]
})
 return df_predictions<save_to_csv>
y_pred_2 = model_mlp.predict(X_train_b_oh)
 rmse = math.sqrt(mean_squared_error(y_train_b, y_pred_2))
R_score = r2_score(y_train_b, y_pred_2)
 print('MLP rmse: ' + str(rmse)+ ', R2: ' + str(R_score))
df_predictions = predict(sample_submission, kaggle_test_loader, lenet)
 df_predictions.to_csv('predictions_lenet.csv', index=False, header=True )<save_to_csv>
y_pred_3 = model_rf.predict(X_train_b)
 rmse = math.sqrt(mean_squared_error(y_train_b, y_pred_3))
R_score = r2_score(y_train_b, y_pred_3)
 print('Random forest rmse: ' + str(rmse)+ ', R2: ' + str(R_score))
df_predictions = predict(sample_submission, kaggle_test_loader, vgg)
 df_predictions.to_csv('predictions_vgg.csv', index=False, header=True )<import_modules>
scale = 70
seed = 7<define_variables>
X_train_b_2 = np.stack([y_pred_1, y_pred_2, y_pred_3], axis=-1)
 model = LinearRegression(n_jobs=-1)
 model.fit(X_train_b_2, y_train_b )
path_to_images = '.. /input/plant-seedlings-classification/train/*/*.png'
images = glob(path_to_images)
 trainingset = []
traininglabels = []
num = len(images)
 count = 1
for i in images:
print(str(count)+'/'+str(num),end='\r')
 trainingset.append(cv2.resize(cv2.imread(i),(scale,scale)))
 traininglabels.append(i.split('/')[-2])
 count=count+1
trainingset = np.asarray(trainingset)
 traininglabels = pd.DataFrame(traininglabels )<categorify>
y_pred_1 = model_xgb.predict(X_train_c)
 y_pred_2 = model_mlp.predict(X_train_c_oh)
 y_pred_3 = model_rf.predict(X_train_c)
 X_train_c_2 = np.stack([y_pred_1, y_pred_2, y_pred_3], axis=-1 )
labels = preprocessing.LabelEncoder()
labels.fit(traininglabels[0])
 print('Classes: '+str(labels.classes_))
encodedlabels = labels.transform(traininglabels[0])
 clearalllabels = np_utils.to_categorical(encodedlabels)
 classes = clearalllabels.shape[1]
print("Number of classes: " + str(classes))
traininglabels[0].value_counts().plot(kind='pie' )<split>
stack_pred = model.predict(X_train_c_2)
 rmse = math.sqrt(mean_squared_error(y_train_c, stack_pred))
R_score = r2_score(y_train_c, y_pred_1)
 print('Ensemble rmse: ' + str(rmse)+ ', R2: ' + str(R_score))
new_train = new_train/255
x_train,x_test,y_train,y_test = train_test_split(new_train,clearalllabels,test_size=0.1,random_state=seed,stratify=clearalllabels)
 print('Train Shape: {}'.format(x_train.shape))<train_model>
X_train_bc_2 = np.concatenate([X_train_b_2, X_train_c_2], axis=0)
 y_train_bc = np.concatenate([y_train_b, y_train_c], axis=0 )
generator = ImageDataGenerator(
 rotation_range = 180,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = True,
vertical_flip = True 
)
 generator.fit(x_train )<choose_model_class>
model.fit(X_train_bc_2, y_train_bc )
np.random.seed(seed)
 model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(5, 5), input_shape=(scale, scale, 3), activation='relu'))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(filters=64, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization(axis=3))
model.add(Dropout(0.1))
model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu'))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization(axis=3))
model.add(Dropout(0.1))
model.add(Conv2D(filters=256, kernel_size=(5, 5), activation='relu'))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(filters=256, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization(axis=3))
model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary() 
<choose_model_class>
test_data.loc[test_data['shop_id'] == 0, 'shop_id'] = 57
test_data.loc[test_data['shop_id'] == 1, 'shop_id'] = 58
test_data.loc[test_data['shop_id'] == 11, 'shop_id'] = 10
test_data['date_block_num'] = 34
lags = [1, 2, 3, 6, 12]
all_data['date_block_num'] = all_data['year_index'] * 12 + all_data['month'] - 1
for lag in lags:
lag_col_name = 'target_lag_' + str(lag)
 shifted = all_data[index_cols + ['target']].copy()
shifted.columns = index_cols + [lag_col_name]
shifted['date_block_num'] += lag
test_data = pd.merge(test_data, shifted, on=index_cols, how='left')
 test_data[lag_col_name].fillna(0, inplace=True)
 test_data = pd.merge(test_data, item_category_mapping, how='left', on='item_id')
 item_id_mean = all_data.groupby('item_id')['target'].mean()
test_data['item_target_enc_exp'] = test_data['item_id'].map(item_id_mean)
 test_data['item_target_enc_exp'].fillna(target_mean, inplace=True)
 shop_id_mean = all_data.groupby('shop_id')['target'].mean()
test_data['shop_target_enc_exp'] = test_data['shop_id'].map(shop_id_mean)
 test_data['shop_target_enc_exp'].fillna(target_mean, inplace=True)
 item_id_mean = all_data.groupby('item_category_id')['target'].mean()
test_data['item_category_target_enc_exp'] = test_data['item_category_id'].map(item_id_mean)
 test_data['item_category_target_enc_exp'].fillna(target_mean, inplace=True)
 test_data = pd.merge(test_data, last_sale_df, on=['shop_id', 'item_id', 'date_block_num'], how='left')
 test_data['last_sale_ago'].fillna(0, inplace=True)
 test_data = pd.merge(test_data, last_shop_sale_df, on=['shop_id', 'date_block_num'], how='left')
 test_data['last_shop_sale_ago'].fillna(0, inplace=True)
 test_data = pd.merge(test_data, last_item_sale_df, on=['item_id', 'date_block_num'], how='left')
 test_data['last_item_sale_ago'].fillna(0, inplace=True)
 test_data['year_index'] = 2
test_data['month'] = 11
test_data['days_in_month'] = 30
test_data.drop(columns='ID', inplace=True)
 test_data.drop(columns='date_block_num', inplace=True)
 test_data.head()
lrr = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.4,
min_lr=0.00001)
 filepath='weights.best_{epoch:02d}-{val_acc:.2f}.h5'
checkpoints = ModelCheckpoint(filepath, monitor='val_acc',
verbose=1, save_best_only=True, mode='max')
 filepath='weights.last_auto4.h5'
checkpoints_full = ModelCheckpoint(filepath, monitor='val_acc',
verbose=1, save_best_only=False, mode='max')
 callbacks_list = [checkpoints, lrr, checkpoints_full]
model.load_weights(".. /input/weights/weights.best_17-0.96.hdf5")
 dataset = np.load(".. /input/plantrecomodels/Data.npz")
 data = dict(zip(( "x_train","x_test","y_train", "y_test"),(dataset[k] for k in dataset)))
 x_train = data['x_train']
x_test = data['x_test']
y_train = data['y_train']
y_test = data['y_test']
print(model.evaluate(x_train, y_train))
print(model.evaluate(x_test, y_test))<save_to_csv>
test_data_oh = test_data.drop(columns=['shop_id', 'item_id', 'item_category_id', 'year_index', 'month', 'last_sale_ago', 'last_shop_sale_ago', 'last_item_sale_ago'])
 for year in range(1,3):
test_data_oh['year_' + str(year)] = int(year == 2)
 for month in range(1, 13):
test_data_oh['month_' + str(month)] = int(month == 11)
 test_data_oh.head()
newtestimages=newtestimages/255
prediction = model.predict(newtestimages)
 pred = np.argmax(prediction,axis=1)
 predStr = labels.classes_[pred]
result = {'file':tests,'species':predStr}
result = pd.DataFrame(result)
 result.to_csv("Prediction.csv",index=False)
 print('Prediction result saved as Prediction.csv' )<import_modules>
X_test_2 = np.stack([model_xgb.predict(test_data), model_mlp.predict(test_data_oh), model_rf.predict(test_data)], axis=-1)
 predictions = model.predict(X_test_2)
 df_pred = pd.DataFrame({'item_cnt_month': predictions})
 df_pred.to_csv('submission.csv', index_label='ID' )