Kaggle competition: House Prices - Advanced Regression Techniques
warnings.filterwarnings('ignore',".*PySoundFile")
 pd.options.display.max_rows = 500
pd.options.display.max_columns = 500<define_variables>
train_all = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col= 'Id')
 test_all = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 target = train_all['SalePrice']
RAW_DATA = ".. /input/birdsong-recognition/"
TRAIN_AUDIO_DIR = ".. /input/birdsong-recognition/train_audio/"
TEST_AUDIO_DIR = ".. /input/birdsong-recognition/test_audio/"<load_from_csv>
train_all.drop(['SalePrice'], axis=1, inplace=True)
 train_all.select_dtypes(exclude='number' ).nunique().sort_values(ascending=True )
train = pd.read_csv(RAW_DATA + "train.csv" )<load_from_csv>
train_all.select_dtypes(exclude='object' ).nunique().sort_values(ascending=True )
if not os.path.exists(TEST_AUDIO_DIR):
print("Run check")
 TEST_AUDIO_DIR = ".. /input/birdcall-check/test_audio/"
test = pd.read_csv(".. /input/birdcall-check/test.csv")
 else:
print("run submission")
 test = pd.read_csv(".. /input/birdsong-recognition/test.csv" )<define_variables>
for col in('BsmtHalfBath', 'HalfBath', 'BsmtFullBath', 'Fireplaces','KitchenAbvGr','FullBath'):
train_all[col] = train_all[col].astype(str)
 test_all[col] = test_all[col].astype(str )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263,'nocall':-1,
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<set_options>
cat_cols = [col for col in train_all.columns if train_all[col].dtype == "object"]
ten_or_over = []
for col in cat_cols:
if train_all[col].nunique() > 10:
ten_or_over.append(col)
 print(ten_or_over )
class Dataset_Test:
def __init__(
 self,
df,
data_dir,
sample_rate=32000,
duration=5,
waveform_transforms=None):
self.data_dir=data_dir
self.clip_frames=sample_rate*duration
self.df=df
self.sample_rate=sample_rate
self.duration=duration
self.audio_ids=df.audio_id.unique()
def __len__(self):
return len(self.audio_ids)
 def __getitem__(self, idx: int):
audio_id=self.audio_ids[idx]
clip, _ = librosa.load(self.data_dir+'{}.mp3'.format(audio_id),
sr=self.sample_rate,
mono=True,
res_type="kaiser_fast")
 clip=clip.astype(np.float32)
 total_frames=len(clip)
 target_df=self.df[self.df.audio_id==audio_id]
all_5s_clips=[]
row_ids=[]
site=target_df.site.iloc[0]
if site=='site_3':
startpoints=list(range(0,total_frames,self.clip_frames))
if(len(startpoints)>1 and total_frames-startpoints[-1]<= 2.5*self.sample_rate):
startpoints.pop(-1)
 for start in startpoints:
clip_5s=clip[start:min(total_frames,start+self.clip_frames)]
if len(clip_5s)<self.clip_frames:
pad0=(self.clip_frames-len(clip_5s)) //2
pad1=self.clip_frames-len(clip_5s)-pad0
clip_5s=np.pad(clip_5s,[pad0,pad1],constant_values=0)
 all_5s_clips.append(clip_5s)
 row_ids.append("site_3_{}".format(audio_id))
else:
for i in range(len(target_df)) :
row=target_df.iloc[i]
end=int(row.seconds*self.sample_rate)
 clip_5s=clip[max(0,end-self.clip_frames):min(total_frames,end)]
if len(clip_5s)<self.clip_frames:
pad0=(self.clip_frames-len(clip_5s)) //2
pad1=self.clip_frames-len(clip_5s)-pad0
clip_5s=np.pad(clip_5s,[pad0,pad1],constant_values=0)
 all_5s_clips.append(clip_5s)
 row_ids.append("{}_{}_{}".format(site,audio_id,int(row.seconds)))
 all_5s_clips=np.stack(all_5s_clips,axis=0)
 return torch.tensor(all_5s_clips),row_ids,site<normalization>
def drop_columns(data, drop_list):
for col in drop_list:
data = data.drop([col], axis = 1)
 return data
train = drop_columns(train_all, ten_or_over)
 test = drop_columns(test_all, ten_or_over)
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./ p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM, self ).__init__()
self.p = nn.Parameter(torch.ones(1)* p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(
 self.eps)+ ')'
class AdaptiveConcatPool2d(nn.Module):
def __init__(self, sz=None,gem=False):
super().__init__()
sz = sz or(1,1)
 self.ap = GeM() if gem else nn.AdaptiveAvgPool2d(sz)
 self.mp = nn.AdaptiveMaxPool2d(sz)
 def forward(self, x):
return torch.cat([self.ap(x), self.mp(x)], 1 ).view(x.size(0),-1)
 class AttMaxPool(nn.Module):

 def __init__(self, n_in,hidden=512,dropout=0,drop_time=0.) :
super().__init__()
att_module = [
nn.Conv1d(n_in, hidden, kernel_size=1, stride=1, padding=0, bias=True),
nn.Tanh() ,
]
if dropout>0:
att_module.append(nn.Dropout(dropout))
att_module.append(nn.Conv1d(hidden,1,kernel_size=1,stride=1,padding=0,bias=True))
self.att=nn.Sequential(*att_module)
 self.drop_time=drop_time
def forward(self, x):
mp=torch.max(x,dim=2)[0]
A=self.att(x)
 A=torch.softmax(A,dim=-1)
 if self.drop_time>0 and self.training:
A=nn.functional.dropout(A,p=self.drop_time,training=self.training)
 A/=A.sum(dim=2,keepdim=True)
 print("drop")
 x = torch.sum(A * x, dim=2)
 x = torch.cat([x,mp],dim=1)
 return x
class ModelWarper(nn.Module):
def __init__(self,arch,dropout=0.5,num_class=264):
super().__init__()
self.basemodel=timm.create_model(arch,pretrained=False)
 features=self.basemodel.get_classifier().in_features
self.pooling=AdaptiveConcatPool2d()
self.dropout=nn.Dropout(dropout)
 self.fc=nn.Linear(2*features,out_features=num_class)
 def forward(self,x):
x=self.basemodel.forward_features(x)
 x=self.pooling(x)
 x=x.view(x.size(0),-1)
 x=self.dropout(x)
 x=self.fc(x)
 return x
class ModelWarper_Att(nn.Module):
def __init__(self,arch,dropout=0.5,dropout_att=0.25,num_class=264,drop_time=0,**kwargs):
super().__init__()
if arch=='PANN_res54':
self.basemodel=ResNet54()
ckpt=torch.load("./model/ResNet54.pth")
 self.basemodel.load_state_dict(ckpt['model'],strict=False)
 features=2048
else:
self.basemodel=timm.create_model(arch,pretrained=False,**kwargs)
 features=self.basemodel.get_classifier().in_features
self.pooling=AttMaxPool(features,dropout=dropout_att,drop_time=drop_time)
 self.dropout=nn.Dropout(dropout)
 self.fc=nn.Linear(2*features,out_features=num_class)
 def forward(self,x):
x=self.basemodel.forward_features(x)
 x=x.mean(dim=2)
 x=self.pooling(x)
 x=self.dropout(x)
 x=self.fc(x)
 return x<define_search_space>
X_train, X_valid, y_train, y_valid = train_test_split(train, target,
train_size=0.9, test_size=0.1, random_state=42)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape
preprocessor=dict()
mel_config = {
"sample_rate": 32000,
"n_fft": [1024,1024,1024],
"hop_length": 313,
"n_mels": 128,
"f_min": 20,
"f_max": 16000,
}
transformer = MelTransformer(**mel_config ).cuda()
augmenter=SpecAugmenter(valid=True,resize=256,crop_height=None)
 preprocessor['standard']={
'transformer':transformer,
'augmenter':augmenter
}
mel_config_2 = {
"sample_rate": 32000,
"n_fft": [1024,1024,1024],
"hop_length": 209,
"n_mels": 128,
"f_min": 20,
"f_max": 16000,
}
transformer = MelTransformer(**mel_config_2 ).cuda()
augmenter=SpecAugmenter(valid=True,resize=320,crop_height=None)
 preprocessor['big']={
'transformer':transformer,
'augmenter':augmenter
}
mel_config_3 = {
"sample_rate": 32000,
"n_fft": [1024,1024,1024],
"hop_length": 313,
"n_mels": 96,
"f_min": 20,
"f_max": 16000,
}
transformer = MelTransformer(**mel_config_3 ).cuda()
augmenter=SpecAugmenter(valid=True,resize=224,crop_height=None)
 preprocessor['small']={
'transformer':transformer,
'augmenter':augmenter
} 
<define_variables>
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import GradientBoostingRegressor
from lightgbm import LGBMRegressor
from sklearn.linear_model import Lasso
from catboost import CatBoostRegressor
from xgboost import XGBRegressor
model_configs=[
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold0_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold1_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold2_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold3_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold4_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold0_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold1_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold2_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold3_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold4_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold0_epoch9.pth",
'base':"regnety_040",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold1_epoch9.pth",
'base':"regnety_040",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold2_epoch9.pth",
'base':"regnety_040",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold3_epoch9.pth",
'base':"regnety_040",
},
]<load_pretrained>
numerical_cols = [col for col in X_train.columns if X_train[col].dtype == "int64" or X_train[col].dtype == "float"]
categorical_cols = [col for col in X_train.columns if X_train[col].dtype == "object"]
numerical_transformer = SimpleImputer(strategy='constant')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
] )
modellist=[]
pplist=[]
for config in model_configs:
model=config['arch'](config['base'],num_class=264)
 print("Loading",config['path'])
 ckpt=torch.load(config['path'])['state_dict']
model.load_state_dict(ckpt)
 model.cuda()
model.eval()
modellist.append(model)
 pplist.append(config['preprocess'])
 print(len(modellist))
print(pplist )<set_options>
preprocess = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
 def process_and_fit(model, train, target):
pipeline = Pipeline(steps=[('preprocess', preprocess), 
('scale', StandardScaler()), 
('model', model)])
 pipeline.fit(train, target)
 return pipeline
def rmse(validation, target):
return np.sqrt(mean_squared_error(validation, target))
torch.cuda.empty_cache()<set_options>
randomForest = RandomForestRegressor(n_estimators=800, random_state=20)
 gradientBoost = GradientBoostingRegressor(n_estimators = 600, random_state=20)
 catBoost = CatBoostRegressor(loss_function='RMSE', random_state=20, verbose=False)
 xgBoost = XGBRegressor(colsample_bytree = 0.3, learning_rate = 0.1, max_depth = 7, alpha = 4, n_estimators = 200)
 lassoReg = Lasso(alpha=0.0005, normalize = True)
 lightGBM = LGBMRegressor(objective='regression' )
!nvidia-smi<load_pretrained>
rfPipeline = process_and_fit(randomForest, X_train, y_train)
 rfPred = rfPipeline.predict(X_valid)
 gbPipeline = process_and_fit(gradientBoost, X_train, y_train)
 gbPred = gbPipeline.predict(X_valid)
 cbPipeline = process_and_fit(catBoost, X_train, y_train)
 cbPred = cbPipeline.predict(X_valid)
 xgbPipeline = process_and_fit(xgBoost, X_train, y_train)
 xgbPred = xgbPipeline.predict(X_valid)
 lassoPipeline = process_and_fit(lassoReg, X_train, y_train)
 lassoPred = lassoPipeline.predict(X_valid)
 gbmPipeline = process_and_fit(lightGBM, X_train, y_train)
 gbmPred = gbmPipeline.predict(X_valid )
dataset=Dataset_Test(
 test,
TEST_AUDIO_DIR,
duration=5,
sample_rate=32000, 
)
 dataloader=DataLoader(dataset,batch_size=1,shuffle=False,num_workers=2,drop_last=False,pin_memory=True )<define_variables>
models = ['Random Forest', 'Gradient Boosting', 'Cat Boost', 'XGBoost', 'Lasso Regression', 'Light GBM']
model_names = [rfPred, gbPred, cbPred, xgbPred, lassoPred, gbmPred]
avg_score = 0
for m in range(len(models)) :
score = rmse(y_valid, model_names[m])
 avg_score += score
print(models[m], "score:", rmse(y_valid, model_names[m]))
print("Average Score Across Models: ", avg_score/6 )
THRESHOLD=0.5
MAX_PRED={
'site_1':3,
'site_2':3,
'site_3':10
}<normalization>
preds_test1 = rfPipeline.predict(test)
 preds_test2 = gbPipeline.predict(test)
 preds_test3 = cbPipeline.predict(test)
 preds_test4 = xgbPipeline.predict(test)
 preds_test5 = lassoPipeline.predict(test)
 preds_test6 = gbmPipeline.predict(test)
 preds_test =(preds_test1+preds_test2+ preds_test3 + preds_test4 + preds_test5 + preds_test6)/6
<create_dataframe><EOS> 
output = pd.DataFrame({'Id': test_all.index,
'SalePrice': preds_test})
 output.to_csv('submission.csv', index=False )