Kaggle competition: Plant Seedlings Classification
<SOS> metric: MeanFScore Kaggle data source: plant-seedlings-classification<data_type_conversions>
%matplotlib inline
use_cuda = torch.cuda.is_available()
use_cuda
data["PoolQC"] = data["PoolQC"].fillna("None")
 data["MiscFeature"] = data["MiscFeature"].fillna("None")
 data["Alley"] = data["Alley"].fillna("None")
 data["Fence"] = data["Fence"].fillna("None")
 data["FireplaceQu"] = data["FireplaceQu"].fillna("None")
 data["LotFrontage"] = data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
data[col] = data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None')
 data["MasVnrType"] = data["MasVnrType"].fillna("None")
 data["MasVnrArea"] = data["MasVnrArea"].fillna(0)
 data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 data = data.drop(['Utilities'], axis=1)
 data["Functional"] = data["Functional"].fillna("Typ")
 data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0])
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0])
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0])
 data['MSSubClass'] = data['MSSubClass'].fillna("None" )<data_type_conversions>
manualSeed = 999
def fixSeed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if use_cuda:
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 fixSeed(manualSeed )
data['MSSubClass'] = data['MSSubClass'].apply(str)
 data['OverallCond'] = data['OverallCond'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str )<categorify>
def find_classes(fullDir):
classes = [d for d in os.listdir(fullDir)if os.path.isdir(os.path.join(fullDir, d)) ]
classes.sort()
class_to_idx = {classes[i]: i for i in range(len(classes)) }
num_to_class = dict(zip(range(len(classes)) , classes))
train = []
for index, label in enumerate(classes):
path = fullDir + label + '/'
for file in listdir(path):
train.append(['{}/{}'.format(label, file), label, index])
 df = pd.DataFrame(train, columns=['file', 'category', 'category_id', ])
 return classes, class_to_idx, num_to_class, df
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(data[c].values))
data[c] = lbl.transform(list(data[c].values))
print('Shape data: {}'.format(data.shape))<feature_engineering>
classes, class_to_idx, num_to_class, df = find_classes(".. /input/train/")
 X, y = df.drop('category_id', axis=1), df['category_id']
print(f'X shape: {X.shape}, y shape: {y.shape}')
 X.head()
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
data[feat] = boxcox1p(data[feat], lam)
<create_dataframe>
class SeedlingDataset(Dataset):
def __init__(self, filenames, labels, root_dir, subset=False, transform=None):
self.filenames = filenames
self.labels = labels
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.labels)
 def __getitem__(self, idx):
fullname = join(self.root_dir, self.filenames.iloc[idx])
 image = Image.open(fullname ).convert('RGB')
 if self.transform:
image = self.transform(image)
 return image, self.labels.iloc[idx]
data_na =(data.isnull().sum() / len(data)) * 100
data_na = data_na.drop(data_na[data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :data_na})
 missing_data.head()<feature_engineering>
from sklearn.model_selection import train_test_split
data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']<categorify>
from sklearn.model_selection import train_test_split
data = pd.get_dummies(data)
 print(data.shape )<split>
X, y = df.drop(['category_id', 'category'], axis=1), df['category_id']
train = data[:ntrain]
test = data[ntrain:]<import_modules>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
 X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=1 )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
def train(train_loader, model, optimizer, criterion):
model.train()
losses, accuracies = [], []
for batch_idx,(data, target)in(enumerate(train_loader)) :
correct = 0
if use_cuda:
data, target = data.cuda() , target.cuda()
optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 loss.backward()
pred = output.data.max(1)[1]
correct += pred.eq(target.data ).cpu().sum()
accuracy = 100.* correct / len(data)
 optimizer.step()
losses.append(loss.data.item())
 accuracies.append(accuracy)
 return np.mean(losses), np.mean(accuracies)
 def test(test_loader, criterion, model):
test_loss = 0
correct = 0
for data, target in test_loader:
if use_cuda:
data, target = data.cuda() , target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
 output = model(data)
 test_loss += criterion(output, target ).data.item()
pred = output.data.max(1)[1]
correct += pred.eq(target.data ).cpu().sum().item()
test_loss /= len(test_loader)
 accuracy = 100.* correct / len(test_loader.dataset)
 return test_loss, accuracy
def _should_early_stop(recent_validation_losses, validation_loss, early_stopping_rounds):
recent_validation_losses.append(validation_loss)
 if early_stopping_rounds < len(recent_validation_losses):
recent_validation_losses.popleft()
return all(np.diff(recent_validation_losses)> 0)
 return False
def run_train_process(epochs, loaders, model, optimizer, criterion, early_stopping_rounds=10):
epoch_to_results = {}
recent_validation_losses = deque()
for epoch in tqdm(range(epochs)) :
train_loss, train_accuracy = train(loaders['train'], model, optimizer, criterion)
 validation_loss, validation_accuracy = test(loaders['validation'], criterion, model)
 epoch_to_results[epoch] = {
'train_loss': train_loss,
'train_accuracy': train_accuracy,
'validation_loss': validation_loss,
'validation_accuracy': validation_accuracy,
}
if epoch % 5 == 0:
print('[%s] Train Accuracy: %.5f , Validation Accuracy: %.5f' %(epoch, train_accuracy, validation_accuracy))
should_early_stop = _should_early_stop(recent_validation_losses, validation_loss, early_stopping_rounds)
 if should_early_stop:
print(f'Train epoch {epoch} EARLY STOPPING - validation loss has not improved in the last {early_stopping_rounds} rounds')
 break
df_epoch_to_results = pd.DataFrame.from_dict(epoch_to_results ).T
_, test_accuracy = test(loaders['test'], criterion, model)
 return df_epoch_to_results, test_accuracy
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from tqdm import tqdm, tqdm_notebook
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))<compute_test_metric>
class LeNet(nn.Module):
def __init__(self, num_classes=12, num_rgb=3):
super(LeNet, self ).__init__()
self.conv1 = nn.Conv2d(num_rgb, 6, 5)
 self.conv2 = nn.Conv2d(6, 16, 5)
 self.fc1 = nn.Linear(44944, 120)
 self.fc2 = nn.Linear(120, 84)
 self.fc3 = nn.Linear(84, num_classes)
 def forward(self, x):
out = F.relu(self.conv1(x))
out = F.max_pool2d(out, 2)
 out = F.relu(self.conv2(out))
out = F.max_pool2d(out, 2)
 out = out.view(out.size(0), -1)
 out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
out = out.view(out.size(0), -1)
 out = self.fc3(out)
 return out
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 image_size =(224, 224)
 train_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor()
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor()
])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 batch_size = 8
loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment2_result, test_accuracy_experiment2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment2_result
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))<compute_test_metric>
image_size =(224, 224)
 train_set = SeedlingDataset(
 X_train.file, y_train, '.. /input/train/',
transform=transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ]))
means = []
means_sq = []
for img, _ in tqdm_notebook(train_set):
means.append(np.asarray(img, dtype='float32' ).mean(axis=(1,2)))
 means_sq.append(( np.asarray(img, dtype='float32')** 2 ).mean(axis=(1,2)))
 mean_img = np.mean(means, axis=0)
 std_img = np.sqrt(np.mean(means_sq, axis=0)-(mean_img ** 2))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 image_size =(224, 224)
 train_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 batch_size = 8
loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment3_part1_result, test_accuracy_experiment3_part1 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment3_part1_result
KRR = KernelRidge(alpha=0.8, kernel='polynomial', degree=2, coef0=3.5 )<compute_test_metric>
image_size =(224, 224)
 batch_size = 8
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment3_part2_result, test_accuracy_experiment3_part2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment3_part2_result
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
class LeNetWithKernelSize3(LeNet):
def __init__(self, num_classes=12, num_rgb=3):
super(LeNetWithKernelSize3, self ).__init__(num_classes, num_rgb)
 self.conv1 = nn.Conv2d(num_rgb, 6, 3)
 self.conv2 = nn.Conv2d(6, 16, 3)
 self.fc1 = nn.Linear(46656, 120)
 self.fc2 = nn.Linear(120, 84)
 self.fc3 = nn.Linear(84, num_classes )
GBoost = GradientBoostingRegressor(n_estimators=1000, learning_rate=0.05,
max_depth=2, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=42 )<compute_test_metric>
image_size =(224, 224)
 batch_size = 8
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNetWithKernelSize3().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 batch_size = 8
loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 3
df_experiment4_part1_result, test_accuracy_experiment4_part1 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment4_part1_result
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNetWithKernelSize3().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 100
df_experiment4_part2_result, test_accuracy_experiment4_part2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment4_part2_result
model_xgb = xgb.XGBRegressor(colsample_bytree=0.5604, gamma=0.0578,
learning_rate=0.05, max_depth=2,
min_child_weight=1.7817, n_estimators=900,
reg_alpha=0.4765, reg_lambda=0.9173,
subsample=0.4738, silent=1,
random_state =42, nthread = -1 )<compute_test_metric>
class LeNetWithSigmoidActivation(LeNet):
def forward(self, x):
out = F.sigmoid(self.conv1(x))
out = F.max_pool2d(out, 2)
 out = F.sigmoid(self.conv2(out))
out = F.max_pool2d(out, 2)
 out = out.view(out.size(0), -1)
 out = F.sigmoid(self.fc1(out))
out = F.sigmoid(self.fc2(out))
out = out.view(out.size(0), -1)
 out = self.fc3(out)
 return out
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNetWithSigmoidActivation().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(lenet.parameters() , lr=learning_rate, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 100
df_experiment5_part1_result, test_accuracy_experiment5_part1 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment5_part1_result
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
lenet = LeNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters() , lr=learning_rate)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 100
df_experiment5_part2_result, test_accuracy_experiment5_part2 = run_train_process(epochs, loaders, lenet, optimizer, criterion)
 df_experiment5_part2_result
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
vgg16 = models.vgg16_bn(pretrained=True)
 print(f'Number of output fearurs of pretrained vgg is: {vgg16.classifier[6].out_features}')
 for param in vgg16.features.parameters() :
param.require_grad = False
num_features = vgg16.classifier[6].in_features
print(f'Last layer of vgg16 has {num_features} input features.')
 print(f'Removing layer: {list(vgg16.classifier.children())[-1]}')
 classifier_layers = list(vgg16.classifier.children())[:-1]
layer_to_add = nn.Linear(num_features, len(np.unique(y)))
 print(f'Adding layer: {layer_to_add}')
 classifier_layers.extend([layer_to_add])
 vgg16.classifier = nn.Sequential(*classifier_layers )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
image_size =(224, 224)
 batch_size = 32
train_trans = transforms.Compose([
transforms.transforms.RandomHorizontalFlip() ,
transforms.transforms.RandomRotation(180),
transforms.transforms.RandomVerticalFlip() ,
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img),
])
 validation_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)
 validation_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0)
 learning_rate = 1e-3
vgg = vgg16.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(vgg.parameters() , lr=0.001, momentum=0.9)
 train_set = SeedlingDataset(X_train.file, y_train, '.. /input/train/', transform=train_trans)
 valid_set = SeedlingDataset(X_val.file, y_val, '.. /input/train/', transform=validation_trans)
 test_set = SeedlingDataset(X_test.file, y_test, '.. /input/train/', transform=validation_trans)
 loaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),
'validation': DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0),
'test': DataLoader(test_set, batch_size=batch_size, shuffle=True, num_workers=0)
 }
epochs = 30
df_experiment8_result, test_accuracy_experiment8 = run_train_process(epochs, loaders, vgg, optimizer, criterion)
 df_experiment8_result
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
df_experiment2_result['experiment'] = 'experiment2'
df_experiment3_part1_result['experiment'] = 'experiment3_part1'
df_experiment3_part2_result['experiment'] = 'experiment3_part2'
df_experiment4_part1_result['experiment'] = 'experiment4_part1'
df_experiment4_part2_result['experiment'] = 'experiment4_part2'
df_experiment5_part1_result['experiment'] = 'experiment5_part1'
df_experiment5_part2_result['experiment'] = 'experiment5_part2'
df_experiment8_result['experiment'] = 'experiment8'
df_all_expirement_results = pd.concat([
df_experiment2_result,
df_experiment3_part1_result,
df_experiment3_part2_result,
df_experiment4_part1_result,
df_experiment4_part2_result,
df_experiment5_part1_result,
df_experiment5_part2_result,
df_experiment8_result
] ).rename_axis('epoch' ).reset_index()
df_all_expirement_results
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
df_all_expirement_results.to_csv('df_all_expirement_results.csv', index=False, header=True )
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
data_dir = '.. /input/'
sample_submission = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
image_size =(224, 224)
 test_trans = transforms.Compose([
transforms.Resize(size=image_size),
transforms.ToTensor() ,
transforms.Normalize(mean_img, std_img)
 ])
 kaggle_train_set = SeedlingDataset(
 sample_submission.file,
sample_submission.species,
'.. /input/test/',
transform=test_trans)
 kaggle_test_loader = DataLoader(kaggle_train_set, batch_size=8, shuffle=False, num_workers=0)
 def predict(sample_submission, kaggle_test_loader, model):
predictions = []
for data, _ in kaggle_test_loader:
if use_cuda:
data = data.cuda()
data = Variable(data)
 output = model(data)
 pred = output.data.max(1)[1]
predictions.extend(pred.tolist())
 df_predictions = pd.DataFrame.from_dict({
'file': sample_submission.file,
'species': [num_to_class[p] for p in predictions]
})
 return df_predictions
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
df_predictions = predict(sample_submission, kaggle_test_loader, lenet)
 df_predictions.to_csv('predictions_lenet.csv', index=False, header=True )
<predict_on_test><EOS> 
df_predictions = predict(sample_submission, kaggle_test_loader, vgg)
 df_predictions.to_csv('predictions_vgg.csv', index=False, header=True )