Kaggle competition: Freesound Audio Tagging 2019
ridgeParamGrid2 = {'alpha' : [1, 2, 3],
'solver': ['svd'],
'normalize' : [True],
'fit_intercept' : [False]}
ridgeGrid2 = GridSearchCV(ridgeRegressor, param_grid = ridgeParamGrid2)
 bestRidge = ridgeGrid2.fit(X_train, y_train)
 PrintPerformance(bestRidge, 'Ridge Regression')
 xgbParamGrid2 = {'learning_rate': [.08,.1,.15],
'gamma' : [10, 20, 30],
'subsample' : [.9,.95, 1],
'max_depth': [5]}
xgbGrid2 = GridSearchCV(xgbRegressor, param_grid = xgbParamGrid2)
 bestXgb = xgbGrid2.fit(X_train, y_train)
 PrintPerformance(bestXgb, 'XGBoost Regression' )<predict_on_test>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
ridgePred = bestRidge.predict(testData)
 gradientPred = bestGradient.predict(testData)
 xgbPred = bestXgb.predict(testData)
 adaPred = bestAda.predict(testData)
 treePred = bestTree.predict(testData)
 submissionRidge = pd.DataFrame({'Id': testData.Id.astype(int), 'SalePrice': ridgePred.astype(int)})
 submissionGradient = pd.DataFrame({'Id': testData.Id.astype(int), 'SalePrice': gradientPred.astype(int)})
 submissionXgb = pd.DataFrame({'Id': testData.Id.astype(int), 'SalePrice': xgbPred.astype(int)})
 submissionAda = pd.DataFrame({'Id': testData.Id.astype(int), 'SalePrice': adaPred.astype(int)})
 submissionTree = pd.DataFrame({'Id': testData.Id.astype(int), 'SalePrice': treePred.astype(int)})
 submissionRidge.to_csv('submissionRidge.csv', index = False)
 submissionGradient.to_csv('submissionGradient.csv', index = False)
 submissionXgb.to_csv('submissionXgb.csv', index = False)
 submissionAda.to_csv('submissionAda.csv', index = False)
 submissionTree.to_csv('submissionTree.csv', index = False)
 print('Files Saved' )<import_modules>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from sklearn.preprocessing import LabelEncoder,StandardScaler
from sklearn.model_selection import train_test_split,cross_val_score
from sklearn.neighbors import LocalOutlierFactor
from sklearn.impute import SimpleImputer
from sklearn.metrics import mean_absolute_error
from sklearn.ensemble import RandomForestRegressor,StackingRegressor,GradientBoostingRegressor
from sklearn.linear_model import LassoCV,RidgeCV,ElasticNetCV
from sklearn.svm import SVR
from xgboost.sklearn import XGBRegressor
import lightgbm as lbg<load_from_csv>
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED = Path('.. /input/fat2019_prep_mels1')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_TRN_NOISY_BEST50S = PREPROCESSED/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
MELS_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'
MELS_TRN_NOISY = PREPROCESSED/'mels_train_noisy.pkl'
MELS_TRN_NOISY_BEST50S = PREPROCESSED/'mels_trn_noisy_best50s.pkl'
MELS_TEST = PREPROCESSED/'mels_test.pkl'
trn_curated_df = pd.read_csv(CSV_TRN_CURATED)
 trn_noisy_df = pd.read_csv(CSV_TRN_NOISY)
 trn_noisy50s_df = pd.read_csv(CSV_TRN_NOISY_BEST50S)
 test_df = pd.read_csv(CSV_SUBMISSION)
 df = pd.concat([trn_curated_df, trn_noisy50s_df], ignore_index=True, sort=True)
 test_df = pd.read_csv(CSV_SUBMISSION)
 X_train = pickle.load(open(MELS_TRN_CURATED, 'rb')) + pickle.load(open(MELS_TRN_NOISY_BEST50S, 'rb'))
train=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv',index_col='Id')
 test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv',index_col='Id')
 y=train['SalePrice']
X=train.append(test )<count_missing_values>
data.show_batch(3 )
def missing(X):
missing_cols={}
for i in range(len(X.columns)) :
if X.iloc[:,i].isnull().sum() >0:
missing_cols[X.iloc[:,i].name]=X.iloc[:,i].isnull().sum()
return missing_cols
X.drop(['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'],axis=1,inplace=True )<categorify>
def lwlrap(y_pred,y_true):
score, weight = calculate_per_class_lwlrap(y_true.cpu().numpy() , y_pred.cpu().numpy())
 lwlrap =(score * weight ).sum()
return torch.from_numpy(np.array(lwlrap))
null_cats=['BsmtQual', 'BsmtCond','BsmtExposure', 'BsmtFinType1','BsmtFinType2',
'GarageType','GarageFinish','GarageQual', 'GarageCond']
for f in null_cats:
X[f].fillna('None',inplace=True)
 null_nums=['MasVnrArea' ,'GarageYrBlt', 'GarageArea',
'GarageCars','BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']
for f in null_nums:
X[f].fillna(0,inplace=True)
 X['LotFrontage']=X.groupby('Neighborhood')['LotFrontage'].transform(lambda x:x.fillna(x.median()))
missing(X )<categorify>
class MixUpCallback(LearnerCallback):
"Callback that creates the mixed-up input and target."
def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):
super().__init__(learn)
 self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y
def on_train_begin(self, **kwargs):
if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)
 def on_batch_begin(self, last_input, last_target, train, **kwargs):
"Applies mixup to `last_input` and `last_target` if `train`."
if not train: return
lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))
lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1 ).max(1)
 lambd = last_input.new(lambd)
 shuffle = torch.randperm(last_target.size(0)).to(last_input.device)
 x1, y1 = last_input[shuffle], last_target[shuffle]
if self.stack_x:
new_input = [last_input, last_input[shuffle], lambd]
else:
new_input =(last_input * lambd.view(lambd.size(0),1,1,1)+ x1 *(1-lambd ).view(lambd.size(0),1,1,1))
if self.stack_y:
new_target = torch.cat([last_target[:,None].float() , y1[:,None].float() , lambd[:,None].float() ], 1)
 else:
if len(last_target.shape)== 2:
lambd = lambd.unsqueeze(1 ).float()
new_target = last_target.float() * lambd + y1.float() *(1-lambd)
 return {'last_input': new_input, 'last_target': new_target}
def on_train_end(self, **kwargs):
if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()
class MixUpLoss(nn.Module):
"Adapt the loss function `crit` to go with mixup."
def __init__(self, crit, reduction='mean'):
super().__init__()
if hasattr(crit, 'reduction'):
self.crit = crit
self.old_red = crit.reduction
setattr(self.crit, 'reduction', 'none')
 else:
self.crit = partial(crit, reduction='none')
 self.old_crit = crit
self.reduction = reduction
def forward(self, output, target):
if len(target.size())== 2:
loss1, loss2 = self.crit(output,target[:,0].long()), self.crit(output,target[:,1].long())
 d =(loss1 * target[:,2] + loss2 *(1-target[:,2])).mean()
else: d = self.crit(output, target)
 if self.reduction == 'mean': return d.mean()
elif self.reduction == 'sum': return d.sum()
return d
def get_old(self):
if hasattr(self, 'old_crit'): return self.old_crit
elif hasattr(self, 'old_red'):
setattr(self.crit, 'reduction', self.old_red)
 return self.crit
def mixup(learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True)-> Learner:
"Add mixup https://arxiv.org/abs/1710.09412 to `learn`."
learn.callback_fns.append(partial(MixUpCallback, alpha=alpha, stack_x=stack_x, stack_y=stack_y))
return learn
Learner.mixup = mixup
label=LabelEncoder()
for feat in cat_cols:
X[feat]=label.fit_transform(X[feat] )<normalization>
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
class Classifier(nn.Module):
def __init__(self, num_classes=1000):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = self.avgpool(x)
 x = self.fc(x)
 return x
X_scaled=pd.DataFrame(StandardScaler().fit_transform(X))
X_scaled.columns=X.columns
X_scaled.index=X.index
X_scaled
y=np.log(y )<split>
def borrowed_model(pretrained=False, **kwargs):
return Classifier(**kwargs)
 f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(data, borrowed_model, pretrained=False, metrics=[lwlrap] ).mixup(stack_y=False)
 learn.unfreeze()
X_scaled.drop('SalePrice',inplace=True,axis=1)
 train=X_scaled[:1451]
test=X_scaled[1451:]
X_train,X_test,y_train,y_test=train_test_split(train, y, test_size=.2, random_state=42 )<train_on_grid>
learn.fit_one_cycle(250, 2e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )
def model(X_train,y_train,X_test):
las=LassoCV(max_iter=1e7,)
 rid=RidgeCV()
enr=ElasticNetCV(max_iter=1e7)
 gbr=GradientBoostingRegressor()
svm=SVR(C= 20, epsilon= 0.008, gamma=0.0003)
 xgb1=XGBRegressor(objective='reg:squarederror',learning_rate=0.02,n_estimators=3000,subsample=0.7,reg_alpha=0.00006)
 lgbm=lbg.LGBMRegressor(boosting_type='gbdt',objective='regression',n_estimators=3000,learning_rate=0.01,max_bin=200,
bagging_fraction=0.75,bagging_freq=5,feature_fraction=0.2)
 estimators=[('ridge', rid),('lasso',las),('ENR',enr),('GBR',gbr),('SVR',svm),('XGB',xgb1),('LGBM',lgbm)]
stack_model=StackingRegressor(estimators=estimators,final_estimator=xgb1)
 stack_model.fit(X_train,y_train)
 y_stack=stack_model.predict(X_test)
 las.fit(X_train,y_train)
 rid.fit(X_train,y_train)
 enr.fit(X_train,y_train)
 gbr.fit(X_train,y_train)
 svm.fit(X_train,y_train)
 xgb1.fit(X_train,y_train)
 lgbm.fit(X_train,y_train)
 y1=las.predict(X_test)
 y2=rid.predict(X_test)
 y3=enr.predict(X_test)
 y4=gbr.predict(X_test)
 y5=svm.predict(X_test)
 y6=xgb1.predict(X_test)
 y7=lgbm.predict(X_test)
 y_pred=0.1*y1+0.1*y2+0.1*y3+0.1*y4+0.1*y5+0.1*y6+0.1*y7+0.3*y_stack
return y_pred<compute_test_metric>
del X_train
X_test = pickle.load(open(MELS_TEST, 'rb'))
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test )
y_pred=model(X_train,y_train,X_test)
 mean_absolute_error(y_pred,y_test )<compute_test_metric>
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()