Kaggle competition: Freesound Audio Tagging 2019
<SOS> metric: weightedlabelrankingaverageprecision Kaggle data source: freesound-audio-tagging-2019<save_to_csv>
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms
submission = pd.DataFrame(predictions,
columns=le2.classes_,
index=test.index)
 submission.to_csv('LGBM - base model.csv', index_label='Id' )<import_modules>
torch.cuda.is_available()
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder as le
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
import matplotlib.pyplot as plt
import seaborn as sns
from collections import OrderedDict
from sklearn.cross_validation import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
import xgboost as xgb
from sklearn.tree import DecisionTreeClassifier 
<define_variables>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 520
seed_everything(SEED )
ROOT_FILENAME = ".. /input/"
TRAIN_FILENAME = 'train.csv'
TEST_FILENAME = 'test.csv'
RESULT_FILENAME = 'res2.cv'<load_from_csv>
N_JOBS = cpu_count()
os.environ['MKL_NUM_THREADS'] = str(N_JOBS)
 os.environ['OMP_NUM_THREADS'] = str(N_JOBS)
 DataLoader = partial(DataLoader, num_workers=N_JOBS )
train = pd.read_csv(ROOT_FILENAME+TRAIN_FILENAME, parse_dates=['Dates'], index_col=False)
 print(train.shape)
 train.head(3 )<load_from_csv>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
test = pd.read_csv(ROOT_FILENAME+TEST_FILENAME, parse_dates=['Dates'], index_col=False)
 print(test.shape)
 test.head(3 )<define_variables>
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )
weekdays = {'Monday':0., 'Tuesday':1., 'Wednesday':2., 'Thursday': 3., 'Friday':4., 'Saturday':5., 'Sunday':6.}
weekdays2 = {'Monday':0., 'Tuesday':0., 'Wednesday':0., 'Thursday': 0., 'Friday':0., 'Saturday':1., 'Sunday':1}<categorify>
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}
def getHourZn(hour):
if(hour >= 2 and hour < 8): return 1;
if(hour >= 8 and hour < 12): return 2;
if(hour >= 12 and hour < 14): return 3;
if(hour >= 14 and hour < 18): return 4;
if(hour >= 18 and hour < 22): return 5;
if(hour < 2 or hour >= 22): return 6;<feature_engineering>
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()
def feature_engineering(data):
data['Day'] = data['Dates'].dt.day
data['Month'] = data['Dates'].dt.month
data['Year'] = data['Dates'].dt.year-2003
data['Hour'] = data['Dates'].dt.hour
data['Minute'] = data['Dates'].dt.minute
data['Day_Num'] = [float(weekdays[w])for w in data.DayOfWeek]
data['WeekOfYear'] = data['Dates'].dt.weekofyear
data['District_Num'] = [float(districts[t])for t in data.PdDistrict]
data['HourZn'] = preprocessing.scale(list(map(getHourZn, data['Dates'].dt.hour)))
 data['isWeekday'] = [float(weekdays2[w])for w in data.DayOfWeek]
data['X'] = preprocessing.scale(list(map(lambda x: x+122.4194, data.X)))
 data['Y'] = preprocessing.scale(list(map(lambda x: x-37.7749, data.Y)))
 data['Address_Type'] = list(map(define_address, data.Address))
return data<split>
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()
X_loc = ['X', 'Y', 'District_Num', 'Address_Type']
X_time = ['Minute', 'Hour']
X_date = ['Year','Month', 'Day', 'Day_Num', 'HourZn']
X_all = X_loc + X_time + X_date
train = feature_engineering(train)
 train['Category_Num'] = [float(categories[t])for t in train.Category]
X_train, X_test, y_train, y_test = train_test_split(train[X_all], train['Category_Num'], test_size = 0.2, random_state = 0)
 test = feature_engineering(test)
<find_best_model_class>
labels = test_df.columns[1:].tolist()
labels
clf = RandomForestClassifier(max_features="log2", max_depth=14, n_estimators=25,
min_samples_split=300, oob_score=False ).fit(X_train,y_train)
 y = clf.predict(X_test)
 recall_score(y,y_test, average='micro')
 recall_score(y,y_test, average='macro')
 recall_score(y,y_test, average='weighted')
 precision_score(y,y_test, average='micro')
 precision_score(y,y_test, average='macro')
 precision_score(y,y_test, average='weighted')
 accuracy_score(y,y_test)
 f1_score(y,y_test, average='micro')
 f1_score(y,y_test, average='macro')
 f1_score(y,y_test, average='weighted')
<compute_train_metric>
num_classes = len(labels)
 num_classes
knn = KNeighborsClassifier(n_neighbors=5)
 knn.fit(X_train, y_train)
 y = knn.predict(X_test)
 recall_score(y,y_test, average='micro')
 recall_score(y,y_test, average='macro')
 recall_score(y,y_test, average='weighted')
 precision_score(y,y_test, average='micro')
 precision_score(y,y_test, average='macro')
 precision_score(y,y_test, average='weighted')
 accuracy_score(y,y_test)
 f1_score(y,y_test, average='micro')
 f1_score(y,y_test, average='macro')
 f1_score(y,y_test, average='weighted')
<compute_train_metric>
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape
lgr = LogisticRegression()
lgr.fit(X_train,y_train)
 y = lgr.predict(X_test)
 recall_score(y,y_test, average='micro')
 recall_score(y,y_test, average='macro')
 recall_score(y,y_test, average='weighted')
 precision_score(y,y_test, average='micro')
 precision_score(y,y_test, average='macro')
 precision_score(y,y_test, average='weighted')
 accuracy_score(y,y_test)
 f1_score(y,y_test, average='micro')
 f1_score(y,y_test, average='macro')
 f1_score(y,y_test, average='weighted')
<load_from_csv>
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )
trainData = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 testData = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 trainData.info()
print('_' * 40)
 testData.info()
trainData.describe()<create_dataframe>
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - base_dim)
 image = image.crop([crop, 0, crop + base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname
def BuildSummary(dataset, column):
meanData = dataset[[column, 'SalePrice']].groupby([column],as_index = False ).mean()
meanData.columns = [column, 'Average Sale Price($)']
countData = dataset[[column, 'SalePrice']].groupby([column],as_index = False ).count()
countData.columns = [column, 'Sale Price Count']
countData = countData.drop([column], axis = 1)
 summary = pd.concat([meanData, countData], axis = 1 ).sort_values(by = 'Average Sale Price($)', ascending = False)
 return summary<create_dataframe>
transforms_dict = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}
listOfSummaryDfs = []
LotFrontage = BuildSummary(trainData, 'LotFrontage')
 LotFrontage.name = 'LotFrontage'
listOfSummaryDfs.append(LotFrontage)
 LotArea = BuildSummary(trainData, 'LotArea')
 LotArea.name = 'LotArea'
listOfSummaryDfs.append(LotArea)
 YearBuilt = BuildSummary(trainData, 'YearBuilt')
 YearBuilt.name = 'YearBuilt'
listOfSummaryDfs.append(YearBuilt)
 YearRemodAdd = BuildSummary(trainData, 'YearRemodAdd')
 YearRemodAdd.name = 'YearRemodAdd'
listOfSummaryDfs.append(YearRemodAdd)
 BsmtFinSF1 = BuildSummary(trainData, 'BsmtFinSF1')
 BsmtFinSF1.name = 'BsmtFinSF1'
listOfSummaryDfs.append(BsmtFinSF1)
 TotalBsmtSF = BuildSummary(trainData, 'TotalBsmtSF')
 TotalBsmtSF.name = 'TotalBsmtSF'
listOfSummaryDfs.append(TotalBsmtSF)
 FirstFlrSF = BuildSummary(trainData, '1stFlrSF')
 FirstFlrSF.name = 'FirstFlrSF'
listOfSummaryDfs.append(FirstFlrSF)
 GrLivArea = BuildSummary(trainData, 'GrLivArea')
 GrLivArea.name = 'GrLivArea'
listOfSummaryDfs.append(GrLivArea)
 TotRmsAbvGrd = BuildSummary(trainData, 'TotRmsAbvGrd')
 TotRmsAbvGrd.name = 'TotRmsAbvGrd'
listOfSummaryDfs.append(TotRmsAbvGrd)
 Fireplaces = BuildSummary(trainData, 'Fireplaces')
 Fireplaces.name = 'Fireplaces'
listOfSummaryDfs.append(Fireplaces)
 GarageYrBlt = BuildSummary(trainData, 'GarageYrBlt')
 GarageYrBlt.name = 'GarageYrBlt'
listOfSummaryDfs.append(GarageYrBlt)
 GarageCars = BuildSummary(trainData, 'GarageCars')
 GarageCars.name = 'GarageCars'
listOfSummaryDfs.append(GarageCars)
 GarageArea = BuildSummary(trainData, 'GarageArea')
 GarageArea.name = 'GarageArea'
listOfSummaryDfs.append(GarageArea)
 WoodDeckSF = BuildSummary(trainData, 'WoodDeckSF')
 WoodDeckSF.name = 'WoodDeckSF'
listOfSummaryDfs.append(WoodDeckSF)
 OpenPorchSF = BuildSummary(trainData, 'OpenPorchSF')
 OpenPorchSF.name = 'OpenPorchSF'
listOfSummaryDfs.append(OpenPorchSF)
 EnclosedPorch = BuildSummary(trainData, 'EnclosedPorch')
 EnclosedPorch.name = 'EnclosedPorch'
listOfSummaryDfs.append(EnclosedPorch)
 PoolArea = BuildSummary(trainData, 'PoolArea')
 PoolArea.name = 'PoolArea'
listOfSummaryDfs.append(PoolArea)
 MoSold = BuildSummary(trainData, 'MoSold')
 MoSold.name = 'MoSold'
listOfSummaryDfs.append(MoSold)
 YrSold = BuildSummary(trainData, 'YrSold')
 YrSold.name = 'YrSold'
listOfSummaryDfs.append(YrSold )<count_missing_values>
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels, downsample = None):
super().__init__()
self.downsample = downsample
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.conv2(out)
 out = F.avg_pool2d(out, 2)
 return out
print(trainData.isnull().sum())
 print('_' * 40)
 print(testData.isnull().sum() )<feature_engineering>
class Classifier(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512)
)
 self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = torch.mean(x, dim=3)
 x, _ = torch.max(x, dim=2)
 x = self.fc(x)
 return x
trainData['label'] = 'train'
testData['label'] = 'test'
combined = pd.concat([trainData, testData])
 s =(combined.dtypes != 'object')
 combinedColumnsToFillMedian = list(s[s].index)
 t =(combined.dtypes == 'object')
 combinedColumnsToFillNoData = list(t[t].index)
 numericalImputer = SimpleImputer(strategy = 'median')
 numiercalCombined = pd.DataFrame(numericalImputer.fit_transform(combined[combinedColumnsToFillMedian]))
numiercalCombined.columns = combined[combinedColumnsToFillMedian].columns
categoricalImputer = SimpleImputer(strategy = 'constant', fill_value = 'Nodata/Na')
 categoricalCombined = pd.DataFrame(categoricalImputer.fit_transform(combined[combinedColumnsToFillNoData]))
categoricalCombined.columns = combined[combinedColumnsToFillNoData].columns
combined = pd.concat([numiercalCombined, categoricalCombined], axis = 1)
<categorify>
Classifier(num_classes=num_classes )
combined = pd.get_dummies(combined, columns = combinedColumnsToFillNoData, drop_first = True)
 trainData = combined[combined['label_train'] == 1]
testData = combined[combined['label_train'] == 0]
trainData = trainData.drop('label_train', axis = 1)
 testData = testData.drop(['label_train', 'SalePrice'], axis = 1)
 print(trainData.isnull().sum())
 print('_' * 40)
 print(testData.isnull().sum() )<split>
def train_model(x_train, y_train, train_transforms):
num_epochs = 97
batch_size = 64
test_batch_size = 256
lr = 1e-3
eta_min = 1e-5
t_max = 10
num_classes = y_train.shape[1]
x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)
 train_dataset = FATTrainDataset(x_train, y_train, train_transforms)
 valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)
 model = Classifier(num_classes=num_classes ).cuda()
criterion = nn.BCEWithLogitsLoss().cuda()
optimizer = Adam(params=model.parameters() , lr=lr, amsgrad=False)
 scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)
 best_epoch = -1
best_lwlrap = 0.
 mb = master_bar(range(num_epochs))
for epoch in mb:
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in train_loader:
preds = model(x_batch.cuda())
 loss = criterion(preds, y_batch.cuda())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( len(x_val), num_classes))
avg_val_loss = 0.
 ')
 for i,(x_batch, y_batch)in enumerate(valid_loader):
preds = model(x_batch.cuda() ).detach()
loss = criterion(preds, y_batch.cuda())
 preds = torch.sigmoid(preds)
 valid_preds[i * test_batch_size:(i+1)* test_batch_size] = preds.cpu().numpy()
avg_val_loss += loss.item() / len(valid_loader)
 score, weight = calculate_per_class_lwlrap(y_val, valid_preds)
 lwlrap =(score * weight ).sum()
scheduler.step()
if(epoch + 1)% 1 == 0:
elapsed = time.time() - start_time
mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')
 if lwlrap > best_lwlrap:
best_epoch = epoch + 1
best_lwlrap = lwlrap
torch.save(model.state_dict() , 'weight_best.pt')
 ')
 return {
'best_epoch': best_epoch,
'best_lwlrap': best_lwlrap,
}
train, test = train_test_split(trainData, test_size =.2)
 X_train = train.drop("SalePrice", axis = 1)
 y_train = train["SalePrice"]
X_test = test.drop("SalePrice", axis = 1)
 y_test = test["SalePrice"]<choose_model_class>
result = train_model(x_train, y_train, transforms_dict['train'] )
lrRegressor = LinearRegression().fit(X_train, y_train)
 lrPred = lrRegressor.predict(X_test)
 ridgeRegressor = Ridge().fit(X_train, y_train)
 ridgePred = ridgeRegressor.predict(X_test)
 treeRegressor = DecisionTreeRegressor().fit(X_train, y_train)
 treePred = treeRegressor.predict(X_test)
 neighborRegressor = KNeighborsRegressor().fit(X_train, y_train)
 knPred = neighborRegressor.predict(X_test)
 baggingRegressor = BaggingRegressor().fit(X_train, y_train)
 baggingPred = baggingRegressor.predict(X_test)
 adaRegressor = AdaBoostRegressor().fit(X_train, y_train)
 adaPred = adaRegressor.predict(X_test)
 gradientRegressor = GradientBoostingRegressor().fit(X_train, y_train)
 gradientPred = gradientRegressor.predict(X_test)
 xgbRegressor = xgb.XGBRegressor().fit(X_train, y_train)
 xgbPred = xgbRegressor.predict(X_test )<train_on_grid>
test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes, tta=35 )
def PrintPerformance(regressor, model):
print(model)
 print('Best Score: ' + str(regressor.best_score_))
print('Best Parameters: ', str(regressor.best_params_), '
')
 lrParamGrid = {'fit_intercept' : [True, False],
'normalize' : [True, False]}
lrGrid = GridSearchCV(lrRegressor, param_grid = lrParamGrid)
 bestLr = lrGrid.fit(X_train, y_train)
 PrintPerformance(bestLr, 'Linear Regression')
 ridgeParamGrid = {'alpha' : [.1,.5, 1, 5, 10, 50, 100],
'fit_intercept' : [True, False],
'normalize' : [True, False],
'solver' : ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']}
ridgeGrid = RandomizedSearchCV(ridgeRegressor, param_distributions = ridgeParamGrid)
 bestRidge = ridgeGrid.fit(X_train, y_train)
 PrintPerformance(bestRidge, 'Ridge Regression')
 treeParamGrid = {'criterion' : ['mse', 'friedman_mse', 'mae'],
'min_samples_leaf' : [1, 2, 3, 5, 8, 10],
'splitter' : ['best', 'random']}
treeGrid = GridSearchCV(treeRegressor, param_grid = treeParamGrid)
 bestTree = treeGrid.fit(X_train, y_train)
 PrintPerformance(bestTree, 'Tree Regression')
 knnParamGrid = {'n_neighbors' : [1, 2, 3, 4, 5],
'algorithm' : ['auto', 'ball_tree', 'kd_tree', 'brute'],
'weights': ['uniform', 'distance']}
knnGrid = RandomizedSearchCV(neighborRegressor, param_distributions = knnParamGrid)
 bestKnn = knnGrid.fit(X_train, y_train)
 PrintPerformance(bestKnn, 'Knn Regression')
 baggingParamGrid = {'n_estimators' : [200],
'max_features' : [1, 2, 3, 5, 8, 10],
'bootstrap': [True, False],
'bootstrap_features': [True, False]}
baggingGrid = RandomizedSearchCV(baggingRegressor, param_distributions = baggingParamGrid)
 bestBagging = baggingGrid.fit(X_train, y_train)
 PrintPerformance(bestBagging, 'Bagging Regression')
 adaBoostParamGrid = {'n_estimators' : [200],
'loss' : ['linear', 'square', 'exponential']}
adaGrid = GridSearchCV(adaRegressor, param_grid = adaBoostParamGrid)
 bestAda = adaGrid.fit(X_train, y_train)
 PrintPerformance(bestAda, 'Ada Regression')
 gradientParamGrid = {'loss' : ['ls', 'lad', 'huber', 'quantile'],
'n_estimators' : [200],
'subsample': [.5,.8, 1],
'criterion' : ['mse', 'friedman_mse', 'mae']}
gradientGrid = RandomizedSearchCV(gradientRegressor, param_distributions = gradientParamGrid)
 bestGradient = gradientGrid.fit(X_train, y_train)
 PrintPerformance(bestGradient, 'Gradient Regression')
 xgbParamGrid = {'learning_rate': [.05,.1,.2,.3,.4,.5],
'gamma' : [0, 1, 5, 10, 50, 100],
'max_depth' : [3, 4, 5, 6, 7, 8],
'subsample' : [.5,.8, 1]}
xgbGrid = RandomizedSearchCV(xgbRegressor, param_distributions = xgbParamGrid)
 bestXgb = xgbGrid.fit(X_train, y_train)
 PrintPerformance(bestXgb, 'XGBoost Regression' )<train_on_grid>
test_df[labels] = test_preds.values
test_df.to_csv('submission.csv', index=False)
 test_df.head()