Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
warnings.filterwarnings('ignore')
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_logger(out_file=None):
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 logger.handlers = []
logger.setLevel(logging.INFO)
 handler = logging.StreamHandler()
handler.setFormatter(formatter)
 handler.setLevel(logging.INFO)
 logger.addHandler(handler)
 if out_file is not None:
fh = logging.FileHandler(out_file)
 fh.setFormatter(formatter)
 fh.setLevel(logging.INFO)
 logger.addHandler(fh)
 logger.info("logger set up")
 return logger
@contextmanager
def timer(name: str, logger: Optional[logging.Logger] = None):
t0 = time.time()
msg = f"[{name}] start"
if logger is None:
print(msg)
 else:
logger.info(msg)
 yield
msg = f"[{name}] done in {time.time() - t0:.2f} s"
if logger is None:
print(msg)
 else:
logger.info(msg )<set_options>
def preprocess_data(verbose=False):
if verbose:
print("*** BEGIN PREPROCESSING ***")
 data_directory = Path(".. /input/house-prices-advanced-regression-techniques/")
 df, train_indices, test_indices = load(data_directory, verbose)
 df = clean(df, verbose)
 df = order_ordinals(df, verbose)
 df = encode_nominative_categoricals(df, verbose)
 df = impute(df, verbose)
 df_train = df.loc[train_indices, :]
df_test = df.loc[test_indices, :]
if verbose:
print("*** END PREPROCESSING ***
")
 return df_train, df_test
logger = get_logger("main.log")
 set_seed(1213 )<define_variables>
def load(data_dir, verbose):
df_train = pd.read_csv(data_dir / "train.csv", index_col="Id")
 df_test = pd.read_csv(data_dir / "test.csv", index_col="Id")
 df = pd.concat([df_train, df_test])
 if verbose:
print("Train and test splits read from CSV and merged.")
 print("Missing values before any preprocessing: ", df.isnull().values.sum())
 return df, df_train.index, df_test.index
TARGET_SR = 32000
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()<load_from_csv>
def clean(df, verbose):
cleaned_features = ["Exterior2nd", "GarageYrBlt"]
df["Exterior2nd"] = df["Exterior2nd"].replace({"Brk Cmn": "BrkComm"})
 df["Exterior2nd"] = df["Exterior2nd"].replace({"Wd Shng": "WdShing"})
 df["Exterior2nd"] = df["Exterior2nd"].replace({"CmentBd": "CemntBd"})
 df["GarageYrBlt"] = df["GarageYrBlt"].where(df.GarageYrBlt <= 2010, df.YearBuilt)
 name_pairs = {"1stFlrSF": "FirstFlrSF",
"2ndFlrSF": "SecondFlrSF",
"3SsnPorch": "ThreeSeaPorch",}
df.rename(columns=name_pairs, inplace=True)
 if verbose:
print("Cleaned: ", cleaned_features, sep='
')
 print("Renamed(From, To): ", *name_pairs.items() , sep='
')
 return df
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdsong-recognition")
 test = pd.read_csv(DATA_DIR / "test.csv")
 test_audio = DATA_DIR / "test_audio"
test.head()<save_to_csv>
five_levels = ["Po", "Fa", "TA", "Gd", "Ex"]
ten_levels = list(range(10))
ordered_levels = {
"OverallQual": ten_levels,
"OverallCond": ten_levels,
"ExterQual": five_levels,
"ExterCond": five_levels,
"BsmtQual": five_levels,
"BsmtCond": five_levels,
"HeatingQC": five_levels,
"KitchenQual": five_levels,
"FireplaceQu": five_levels,
"GarageQual": five_levels,
"GarageCond": five_levels,
"PoolQC": five_levels,
"LotShape": ["Reg", "IR1", "IR2", "IR3"],
"LandSlope": ["Sev", "Mod", "Gtl"],
"BsmtExposure": ["No", "Mn", "Av", "Gd"],
"BsmtFinType1": ["Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"],
"BsmtFinType2": ["Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"],
"Functional": ["Sal", "Sev", "Maj2", "Maj1", "Mod", "Min2", "Min1", "Typ"],
"GarageFinish": ["Unf", "RFn", "Fin"],
"PavedDrive": ["N", "P", "Y"],
"Utilities": ["NoSeWa", "NoSewr", "AllPub"],
"CentralAir": ["N", "Y"],
"Electrical": ["Mix", "FuseP", "FuseF", "FuseA", "SBrkr"],
"Fence": ["MnPrv", "MnWw", "GdWo", "GdPrv"],
}
ordered_levels = {key: ["None"] + value for key, value in
ordered_levels.items() }
def print_n_per_line(items, n):
for idx, item in enumerate(items):
print(" " + item + ''.join([' ' for i in range(14 - len(item)) ]), end='')
 if idx % n == n-1:
print()
print()
def order_ordinals(df, verbose):
for name, levels in ordered_levels.items() :
df[name] = df[name].astype(CategoricalDtype(levels, ordered=True))
if verbose:
print("Created ordinal categories and ordered with levels: ")
 ')
 print_n_per_line(list(ordered_levels.keys()), 5)
 return df
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )<choose_model_class>
features_nom = ["MSSubClass", "MSZoning", "Street", "Alley", "LandContour", "LotConfig", "Neighborhood", "Condition1", "Condition2", "BldgType", "HouseStyle", "RoofStyle", "RoofMatl", "Exterior1st", "Exterior2nd", "MasVnrType", "Foundation", "Heating", "CentralAir", "GarageType", "MiscFeature", "SaleType", "SaleCondition"]
def encode_nominative_categoricals(df, verbose):
for name in features_nom:
df[name] = df[name].astype("category")
 if "None" not in df[name].cat.categories:
df[name].cat.add_categories("None", inplace=True)
 if verbose:
print("Created nominative categories(mostly conversions from numericals): ")
 '
print_n_per_line(features_nom, 5)
 return df
class ResNet(nn.Module):
def __init__(self, base_model_name: str, pretrained=False,
num_classes=264):
super().__init__()
base_model = models.__getattribute__(base_model_name )(
 pretrained=pretrained)
 layers = list(base_model.children())[:-2]
layers.append(nn.AdaptiveMaxPool2d(1))
self.encoder = nn.Sequential(*layers)
 in_features = base_model.fc.in_features
self.classifier = nn.Sequential(
 nn.Linear(in_features, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, num_classes))
def forward(self, x):
batch_size = x.size(0)
 x = self.encoder(x ).view(batch_size, -1)
 x = self.classifier(x)
 multiclass_proba = F.softmax(x, dim=1)
 multilabel_proba = F.sigmoid(x)
 return {
"logits": x,
"multiclass_proba": multiclass_proba,
"multilabel_proba": multilabel_proba
}<define_variables>
def impute(df, verbose):
if verbose:
print("
Missing values before imputing:")
 sums_df = pd.DataFrame(df.isnull().sum() , columns = ['Missing Values'])
 print(sums_df[sums_df['Missing Values'] > 0])
 total_missing_values = df.isnull().values.sum()
print("Total missing values: ", total_missing_values)
 df['MSZoning'] = df.groupby("Neighborhood")['MSZoning'].transform(lambda x: x.fillna(x.mode()))
df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode() [0])
 df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode() [0])
 df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode() [0])
 for name in df.select_dtypes("number"):
df[name] = df[name].fillna(0)
 for name in df.select_dtypes("category"):
df[name] = df[name].fillna("None")
 if verbose:
print("
missing values imputed: ", total_missing_values,)
 print("missing values remaining after imputing: ", df.isnull().values.sum())
 return df
model_config = {
"base_model_name": "resnet50",
"pretrained": False,
"num_classes": 264
}
melspectrogram_parameters = {
"n_mels": 128,
"fmin": 20,
"fmax": 16000
}
weights_path = ".. /input/birdcall-resnet50-init-weights/best.pth"<define_variables>
df_train, df_test = preprocess_data(verbose=False )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<choose_model_class>
def score_dataset(X, y, model=XGBRegressor()):
for colname in X.select_dtypes(["category"]):
X[colname] = X[colname].cat.codes
log_y = np.log(y)
 score = cross_val_score(
 model, X, log_y, cv=5, scoring="neg_mean_squared_error")
 score = -1 * score.mean()
score = np.sqrt(score)
 return score
def get_model(config: dict, weights_path: str):
model = ResNet(**config)
 checkpoint = torch.load(weights_path)
 model.load_state_dict(checkpoint["model_state_dict"])
 device = torch.device("cuda")
 model.to(device)
 model.eval()
return model<create_dataframe>
X = df_train.copy()
y = X.pop("SalePrice")
 baseline_score = score_dataset(X, y)
 print(f"Baseline score: {baseline_score:.5f} RMSLE" )
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
model: ResNet,
mel_params: dict,
threshold=0.55):
dataset = TestDataset(df=test_df,
clip=clip,
img_size=224,
melspectrogram_parameters=mel_params)
 loader = data.DataLoader(dataset, batch_size=1, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.eval()
prediction_dict = {}
for image, row_id, site in progress_bar(loader):
site = site[0]
row_id = row_id[0]
if site in {"site_1", "site_2"}:
image = image.to(device)
 with torch.no_grad() :
prediction = model(image)
 proba = prediction["multilabel_proba"].detach().cpu().numpy().reshape(-1)
 events = proba >= threshold
labels = np.argwhere(events ).reshape(-1 ).tolist()
else:
image = image.squeeze(0)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
all_events = set()
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
prediction = model(batch)
 proba = prediction["multilabel_proba"].detach().cpu().numpy()
events = proba >= threshold
for i in range(len(events)) :
event = events[i, :]
labels = np.argwhere(event ).reshape(-1 ).tolist()
for label in labels:
all_events.add(label)
 labels = list(all_events)
 if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string
return prediction_dict<load_pretrained>
X = df_train.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 mi_scores
def prediction(test_df: pd.DataFrame,
test_audio: Path,
model_config: dict,
mel_params: dict,
weights_path: str,
threshold=0.5):
model = get_model(model_config, weights_path)
 unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}", logger):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=TARGET_SR,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}", logger):
prediction_dict = prediction_for_clip(test_df_for_audio_id,
clip=clip,
model=model,
mel_params=mel_params,
threshold=threshold)
 row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 prediction_df = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df<save_to_csv>
def drop_uninformative(df, mi_scores, verbose=False):
if verbose:
print('Dropping the following features with low mi_scores:')
 print(df.loc[:, mi_scores == 0.0].columns)
 return df.loc[:, mi_scores > 0.0]
submission = prediction(test_df=test,
test_audio=test_audio,
model_config=model_config,
mel_params=melspectrogram_parameters,
weights_path=weights_path,
threshold=0.85)
 submission.to_csv("submission.csv", index=False )<install_modules>
X = df_train.copy()
y = X.pop("SalePrice")
 X = drop_uninformative(X, mi_scores, verbose=True)
 score_dataset(X, y )
!pip install.. /input/geffnet-pack/gen-efficientnet-pytorch-master/ > /dev/null
!pip install.. /input/resnest-git/ResNeSt-master/ > /dev/null<import_modules>
def label_encode_these(df, cols=[], verbose=False):
X = df.copy()
for colname in cols:
X[colname] = X[colname].cat.codes
if verbose:
print(len(cols), "categorical features label encoded:",)
 print_n_per_line(cols, 5)
 return X
import math
from glob import glob
from os.path import join as pjoin
from tqdm.notebook import tqdm
from typing import Mapping, Any, List, Tuple, Optional, Union
from pathlib import Path
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import pandas as pd
import librosa
import geffnet
from resnest.torch import resnest50_fast_1s1x64d
from torchvision.models.resnet import ResNet, Bottleneck<define_variables>
def one_hot_encode_except(df, label_encoded_cols=[], verbose=False):
X = df.copy()
remaining_col_names = []
for col_name in list(X):
if X[col_name].dtype.name == 'category' and col_name not in label_encoded_cols:
remaining_col_names.append(col_name)
 X = pd.get_dummies(X, columns=remaining_col_names)
 if verbose:
print("One Hot encoding applied to remaining", len(remaining_col_names), "categorical features:")
 print_n_per_line(remaining_col_names, 5)
 return X
TARGET_SR=32_000
PP_NORMALIZE=True
MIN_SEC = 1
DEVICE = 'cuda'
BATCH_SIZE = 64
SIGMOID_THRESH = 0.3
MAX_BIRDS = None<define_variables>
def mathematical_transforms(df, verbose=False):
X = pd.DataFrame()
X["LivLotRatio"] = df.GrLivArea / df.LotArea
X["Spaciousness"] =(df.FirstFlrSF + df.SecondFlrSF)/ df.TotRmsAbvGrd
X["TotalSF"] = df.TotalBsmtSF + df.FirstFlrSF + df.SecondFlrSF
new_features = ["LivLotRatio", "Spaciousness", "TotalSF"]
if verbose and new_features:
print(len(new_features), "new features created via mathematical transforms: ")
 print(" ", new_features)
 return X
def interactions(df, verbose=False):
new_interactions = ["Bldg with GrLivArea"]
columns_to_interact_with_GrLivArea = ['BldgType']
new_prefixes = ['Bldg']
X = pd.get_dummies(df[columns_to_interact_with_GrLivArea], columns=columns_to_interact_with_GrLivArea, prefix=new_prefixes)
 X = X.mul(df.GrLivArea, axis=0)
 if verbose and new_interactions:
print(len(new_interactions), "new interaction features created: ")
 print(" ", new_interactions, " :")
 for prefix in new_prefixes:
print_n_per_line(list(X.columns[X.columns.str.startswith(prefix)]), 5)
 return X
def counts(df, verbose=False):
X = pd.DataFrame()
X["PorchTypes"] = df[[
"WoodDeckSF",
"OpenPorchSF",
"EnclosedPorch",
"ThreeSeaPorch",
"ScreenPorch",
]].gt(0.0 ).sum(axis=1)
 new_counted_features = ["PorchTypes"]
if verbose and new_counted_features:
print(len(new_counted_features), 'new "feature count" type features created: ')
 print(" ", new_counted_features)
 return X
def break_down(df, verbose=False):
X = pd.DataFrame()
X["MSClass"] = df.MSSubClass.str.split("_", n=1, expand=True)[0]
return X
def group_transforms(df, verbose=False):
X = pd.DataFrame()
X["MedNeighLivArea"] = df.groupby("Neighborhood")["GrLivArea"].transform("median")
 X["MedNeighLotArea"] = df.groupby("Neighborhood")["LotArea"].transform("median")
 X["MedQualLivArea"] = df.groupby("OverallQual")["GrLivArea"].transform("median")
 new_transforms = ["MedNeighLivArea", "MedNeighLotArea", "MedQualLivArea"]
if verbose and new_transforms:
print(len(new_transforms), "new features created via groupby transform of a feature: ")
 print(" ", new_transforms)
 return X
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
CODE2BIRD = {v:k for k,v in BIRD_CODE.items() }<normalization>
def apply_pca(X, standardize=True):
if standardize:
X =(X - X.mean(axis=0)) / X.std(axis=0)
 pca = PCA()
X_pca = pca.fit_transform(X)
 component_names = [f"PC{i+1}" for i in range(X_pca.shape[1])]
X_pca = pd.DataFrame(X_pca, columns=component_names)
 loadings = pd.DataFrame(
 pca.components_.T,
columns=component_names,
index=X.columns, 
)
 return pca, X_pca, loadings
def plot_variance(pca, width=8, dpi=100):
fig, axs = plt.subplots(1, 2)
 n = pca.n_components_
grid = np.arange(1, n + 1)
 evr = pca.explained_variance_ratio_
axs[0].bar(grid, evr)
 axs[0].set(
 xlabel="Component", title="% Explained Variance", ylim=(0.0, 1.0)
)
 cv = np.cumsum(evr)
 axs[1].plot(np.r_[0, grid], np.r_[0, cv], "o-")
 axs[1].set(
 xlabel="Component", title="% Cumulative Variance", ylim=(0.0, 1.0)
)
 fig.set(figwidth=8, dpi=100)
 return axs
class ReadAudio(object):
def __init__(
 self 
):
pass
def __call__(self, filename: str)-> Tuple[np.ndarray, int]:
au, sr = librosa.load(filename, sr=None, dtype=np.float64)
 return au, sr
class PreprocessWaveform(object):
def __init__(
 self,
target_sr: int,
normalize: bool = True, 
):
self.target_sr=target_sr
self.normalize=normalize
def __call__(self, wave: np.ndarray, sr: int)-> np.ndarray:
au = librosa.resample(wave, sr, self.target_sr)
 if self.normalize:
au = librosa.util.normalize(au)
 return au<feature_engineering>
def pca_inspired(df, verbose=False):
X = pd.DataFrame()
X["TotSqFt"] = df.GrLivArea + df.TotalBsmtSF
X["Feature2"] = df.YearRemodAdd * df.TotalBsmtSF
return X
def pca_components(df, features):
X = df.loc[:, features]
_, X_pca, _ = apply_pca(X)
 return X_pca
pca_features = [
"GarageArea",
"YearRemodAdd",
"TotalBsmtSF",
"GrLivArea",]
class InfernceDataPipelineFileName2Au(object):
def __init__(
 self,
target_sr: bool,
pp_normalize: bool, 
):
self.au_reader = ReadAudio()
self.au_pp = PreprocessWaveform(
 target_sr=target_sr,
normalize=pp_normalize 
)
 def __call__(
 self,
au_path: str 
):
readed_au, readed_sr = self.au_reader(au_path)
 au = self.au_pp(readed_au, readed_sr)
 pp_sr = self.au_pp.target_sr
result = {
'au':au,
'sr':pp_sr,
}
return result<create_dataframe>
def indicate_outliers(df, verbose=False):
X_new = pd.DataFrame()
X_new["Outlier"] =(df.Neighborhood == "Edwards")&(df.SaleCondition == "Partial")
 return X_new
class Test2StepDataset(torch.utils.data.Dataset):
def __init__(
 self,
df: pd.DataFrame,
root_path: str,
data_pipeline_austep: object,
extension: str = '.mp3',
duration: int = 5 
):
self.df = df
self.data_pipeline_austep = data_pipeline_austep
self.root_path = root_path
self.extension = extension
self.duration = duration
self.dumped_features = None
def __len__(self):
return len(self.df)
 def __getitem__(self, idx: int):
sample = self.df.loc[idx, :]
site = sample.site
row_id = sample.row_id
name = sample.audio_id
if self.dumped_features is None or self.dumped_features['name'] != name:
self.dumped_features = {
'name': name,
'features': self.data_pipeline_austep(pjoin(self.root_path, name + self.extension))
}
print(f'{name} feature loaded')
 if site == "site_3":
n_secs = len(self.dumped_features['features']['au'])/ self.dumped_features['features']['sr']
sample_rate = self.dumped_features['features']['sr']
start_sec = 0
specs = []
names = []
sites = []
row_ids = []
while n_secs >= start_sec:
y_batch = self.dumped_features['features']['au']
y_batch = y_batch[start_sec*sample_rate:(start_sec+self.duration)*sample_rate]
y_batch = y_batch.astype(np.float32)
 if len(y_batch)< MIN_SEC * sample_rate:
break
if len(y_batch)< sample_rate * self.duration:
y_pad = np.zeros(sample_rate * self.duration, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
y_batch = y_pad
y_batch = torch.from_numpy(y_batch)
 start_sec += self.duration
specs.append(y_batch)
 names.append(name)
 sites.append(sites)
 row_ids.append(row_id)
 return specs, names, sites, row_ids
else:
end_seconds = int(sample.seconds)
 start_seconds = int(end_seconds - self.duration)
 sample_rate = self.dumped_features['features']['sr']
y_batch = self.dumped_features['features']['au']
y_batch = y_batch[start_seconds*sample_rate:end_seconds*sample_rate]
y_batch = y_batch.astype(np.float32)
 if len(y_batch)< sample_rate * self.duration:
y_pad = np.zeros(sample_rate * self.duration, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
y_batch = y_pad
y_batch = torch.from_numpy(y_batch)
 return y_batch, name, site, row_id<load_from_csv>
class CrossFoldEncoder:
def __init__(self, encoder, **kwargs):
self.encoder_ = encoder
self.kwargs_ = kwargs
self.cv_ = KFold(n_splits=5)
 def fit_transform(self, X, y, cols):
self.fitted_encoders_ = []
self.cols_ = cols
X_encoded = []
for idx_encode, idx_train in self.cv_.split(X):
fitted_encoder = self.encoder_(cols=cols, **self.kwargs_)
 fitted_encoder.fit(
 X.iloc[idx_encode, :], y.iloc[idx_encode], 
)
 X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])
 self.fitted_encoders_.append(fitted_encoder)
 X_encoded = pd.concat(X_encoded)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded
def transform(self, X):
X_encoded_list = []
for fitted_encoder in self.fitted_encoders_:
X_encoded = fitted_encoder.transform(X)
 X_encoded_list.append(X_encoded[self.cols_])
 X_encoded = reduce(
 lambda x, y: x.add(y, fill_value=0), X_encoded_list 
)/ len(X_encoded_list)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdcall-check/")
 test_df = pd.read_csv(DATA_DIR / "test.csv")
 test_root = DATA_DIR / "test_audio"
test_df.head()<create_dataframe>
def drop_unhelpful_features(df, features_to_drop=None, verbose=False):
if features_to_drop is None:
return df
else:
if verbose:
print(len(features_to_drop), "features dropped: ")
 print(" ", features_to_drop)
 return df.drop(features_to_drop, axis=1)
test_dataset = Test2StepDataset(
 df=test_df,
root_path=test_root,
data_pipeline_austep=InfernceDataPipelineFileName2Au(
 target_sr=TARGET_SR,
pp_normalize=PP_NORMALIZE 
),
extension='.mp3',
duration=5
)<normalization>
def create_features(df, df_test=None, verbose=False):
if verbose:
print("*** BEGIN FEATURE ENGINEERING PIPELINE ***
")
 X = df.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 if df_test is not None:
X_test = df_test.copy()
X_test.pop("SalePrice")
 X = pd.concat([X, X_test])
 X = X.join(mathematical_transforms(X, verbose))
X = X.join(interactions(X, verbose))
X = X.join(counts(X, verbose))
X = X.join(group_transforms(X, verbose))
X = X.join(pca_inspired(X, verbose))
if df_test is not None:
X_test = X.loc[df_test.index, :]
X.drop(df_test.index, inplace=True)
 encoder = CrossFoldEncoder(MEstimateEncoder, m=1)
 columns_to_encode = ["MSSubClass", "Neighborhood", "YrSold", ]
X = X.join(encoder.fit_transform(X, y, cols=columns_to_encode))
if df_test is not None:
X_test = X_test.join(encoder.transform(X_test))
if verbose:
print(len(columns_to_encode), "features target encoded(with crossfold validation): ")
 print(" ", columns_to_encode)
 if df_test is not None:
X_test = X_test.copy()
X = pd.concat([X, X_test])
 X = drop_unhelpful_features(X, features_to_drop=['Street', 'ThreeSeaPorch', 'Condition2'], verbose=verbose)
 label_encode_columns = list(X.select_dtypes(["category"]))
X = label_encode_these(X, cols=label_encode_columns, verbose=verbose)
 X = one_hot_encode_except(X, label_encoded_cols=label_encode_columns, verbose=verbose)
 if df_test is not None:
X_test = X.loc[df_test.index, :]
X.drop(df_test.index, inplace=True)
 if verbose:
print("*** END FEATURE ENGINEERING PIPELINE ***
")
 if df_test is not None:
return X, X_test
else:
return X
def compute_deltas(
 specgram: torch.Tensor,
win_length: int = 5,
mode: str = "replicate" 
)-> torch.Tensor:
r
device = specgram.device
dtype = specgram.dtype
shape = specgram.size()
specgram = specgram.reshape(1, -1, shape[-1])
 assert win_length >= 3
n =(win_length - 1)// 2
denom = n *(n + 1)*(2 * n + 1)/ 3
specgram = torch.nn.functional.pad(specgram,(n, n), mode=mode)
 kernel = torch.arange(-n, n + 1, 1, device=device, dtype=dtype ).repeat(specgram.shape[1], 1, 1)
 output = torch.nn.functional.conv1d(specgram, kernel, groups=specgram.shape[1])/ denom
output = output.reshape(shape)
 return output
def make_delta(
 input_tensor: torch.Tensor 
):
input_tensor = input_tensor.transpose(3,2)
 input_tensor = compute_deltas(input_tensor)
 input_tensor = input_tensor.transpose(3,2)
 return input_tensor<choose_model_class>
mi_scores = make_mi_scores(X_train, y)
 mi_scores
def _resnext(block, layers, pretrained, progress, **kwargs):
return ResNet(block, layers, **kwargs)
 def resnext50_32x4d_swsl(progress=True, **kwargs):

 kwargs['groups'] = 32
kwargs['width_per_group'] = 4
return _resnext(Bottleneck, [3, 4, 6, 3], True, progress, **kwargs)
 def resnext101_32x4d_swsl(progress=True, **kwargs):

 kwargs['groups'] = 32
kwargs['width_per_group'] = 4
return _resnext(Bottleneck, [3, 4, 23, 3], True, progress, **kwargs )<init_hyperparams>
EFFNETB6_EMB_DIM = 2304
EFFNETB5_EMB_DIM = 2048
EFFNETB4_EMB_DIM = 1792
EFFNETB3_EMB_DIM = 1536
EFFNETB1_EMB_DIM = 1280
RESNEST50_FAST_EMB_DIM = 2048
RESNEXT50_EMB_DIM = 2048
RESNEXT101_EMB_DIM = 2048
EPS = 1e-6
class SpectralEffnet(nn.Module):
def __init__(
 self,
sample_rate: int,
window_size: int,
hop_size: int,
mel_bins: int,
fmin: int,
fmax: int,
top_db: float,
classes_num: int,
encoder_type: str,
hidden_dims: int,
first_dropout_rate: float,
second_dropout_rate: float,
use_pretrained_encoder: bool,
classifier_type: str,
use_spectral_cutout: bool,
spec_aggreagation: str,
use_loudness: bool,
use_spectral_centroid: bool,
device: str,
use_sigmoid: bool = True, 
):
super().__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
self.interpolate_ratio = 32
self.spectrogram_extractor = Spectrogram(
 n_fft=window_size,
hop_length=hop_size,
win_length=window_size,
window=window,
center=center,
pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(
 sr=sample_rate,
n_fft=window_size,
n_mels=mel_bins,
fmin=fmin,
fmax=fmax,
ref=ref,
amin=amin,
top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(
 time_drop_width=64,
time_stripes_num=2,
freq_drop_width=8,
freq_stripes_num=2)
 if use_loudness:
self.loudness_bn = nn.BatchNorm1d(1)
 self.loudness_extractor = Loudness(
 sr=sample_rate,
n_fft=window_size,
min_db=120,
device=device 
)
 if use_spectral_centroid:
self.spectral_centroid_bn = nn.BatchNorm1d(1)
 self.bn0 = nn.BatchNorm2d(mel_bins)
 if spec_aggreagation in ['conv1', 'repeat3', 'deltas', 'conv3', 'time_freq_encoding']:
self.spec_aggreagation = spec_aggreagation
else:
raise ValueError('Invalid spec_aggreagation')
 if encoder_type == 'effnet1':
self.encoder = geffnet.tf_efficientnet_b1_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB1_EMB_DIM
elif encoder_type == 'effnet3':
self.encoder = geffnet.tf_efficientnet_b3_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB3_EMB_DIM
elif encoder_type == 'effnet4':
self.encoder = geffnet.tf_efficientnet_b4_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB4_EMB_DIM
elif encoder_type == 'effnet5':
self.encoder = geffnet.tf_efficientnet_b5_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB5_EMB_DIM
elif encoder_type == 'effnet6':
self.encoder = geffnet.tf_efficientnet_b6_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB6_EMB_DIM
elif encoder_type == 'resnest50_fast_1s1x64d':
if self.spec_aggreagation == 'conv1':
raise ValueError('Invalid spec_aggreagation for this encoder')
 self.encoder = resnest50_fast_1s1x64d(pretrained=use_pretrained_encoder)
 self.encoder.fc = nn.Identity()
nn_embed_size = RESNEST50_FAST_EMB_DIM
elif encoder_type == 'resnext50_32x4d_swsl':
self.encoder = resnext50_32x4d_swsl()
if use_pretrained_encoder:
self.encoder.load_state_dict(
 torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext50_32x4d_swsl' ).state_dict() 
)
 self.encoder.fc = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv1 = nn.Conv2d(
 in_channels=1,
out_channels=self.encoder.conv1.out_channels,
kernel_size=self.encoder.conv1.kernel_size,
stride=self.encoder.conv1.stride,
bias=self.encoder.conv1.bias 
)
 nn_embed_size = RESNEXT50_EMB_DIM
elif encoder_type == 'resnext101_32x4d_swsl':
self.encoder = resnext101_32x4d_swsl()
if use_pretrained_encoder:
self.encoder.load_state_dict(
 torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext101_32x4d_swsl' ).state_dict() 
)
 self.encoder.fc = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv1 = nn.Conv2d(
 in_channels=1,
out_channels=self.encoder.conv1.out_channels,
kernel_size=self.encoder.conv1.kernel_size,
stride=self.encoder.conv1.stride,
bias=self.encoder.conv1.bias 
)
 nn_embed_size = RESNEXT101_EMB_DIM
else:
raise ValueError(f'{encoder_type} is invalid model_type')
 if classifier_type == 'relu':
self.classifier = nn.Sequential(
 nn.Linear(nn_embed_size, hidden_dims), nn.ReLU() , nn.Dropout(p=first_dropout_rate),
nn.Linear(hidden_dims, hidden_dims), nn.ReLU() , nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'elu':
self.classifier = nn.Sequential(
 nn.Dropout(first_dropout_rate),
nn.Linear(nn_embed_size, hidden_dims),
nn.ELU() ,
nn.Dropout(second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'dima':
self.classifier = nn.Sequential(
 nn.BatchNorm1d(nn_embed_size),
nn.Linear(nn_embed_size, hidden_dims),
nn.BatchNorm1d(hidden_dims),
nn.PReLU(hidden_dims),
nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'prelu':
self.classifier = nn.Sequential(
 nn.Dropout(first_dropout_rate),
nn.Linear(nn_embed_size, hidden_dims),
nn.PReLU(hidden_dims),
nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'multiscale_relu':
self.big_dropout = nn.Dropout(p=0.5)
 self.classifier = nn.Sequential(
 nn.Linear(nn_embed_size, hidden_dims), nn.ReLU() , nn.Dropout(p=first_dropout_rate),
nn.Linear(hidden_dims, hidden_dims), nn.ReLU() , nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 else:
raise ValueError("Invalid classifier_type")
 self.classifier_type = classifier_type
self.use_spectral_cutout = use_spectral_cutout
self.use_sigmoid = use_sigmoid
self.use_spectral_centroid = use_spectral_centroid
self.use_loudness = use_loudness
self.encoder_type = encoder_type
self.device = device
self.mel_bins = mel_bins
self.to(self.device)
 def _add_frequency_encoding(self, x):
n, d, h, w = x.size()
vertical = torch.linspace(-1, 1, w, device=x.device ).view(1, 1, 1, -1)
 vertical = vertical.repeat(n, 1, h, 1)
 return vertical
def _add_time_encoding(self, x):
n, d, h, w = x.size()
horizontal = torch.linspace(-1, 1, h, device=x.device ).view(1, 1, -1, 1)
 horizontal = horizontal.repeat(n, 1, 1, w)
 return horizontal
def preprocess(self, input):
x = self.spectrogram_extractor(input)
 additional_features = []
if self.use_loudness:
loudness = self.loudness_extractor(x)
 loudness = self.loudness_bn(loudness)
 loudness = loudness.unsqueeze(-1)
 loudness = loudness.repeat(1,1,1,self.mel_bins)
 additional_features.append(loudness)
 if self.use_spectral_centroid:
spectral_centroid = x.mean(-1)
 spectral_centroid = self.spectral_centroid_bn(spectral_centroid)
 spectral_centroid = spectral_centroid.unsqueeze(-1)
 spectral_centroid = spectral_centroid.repeat(1,1,1,self.mel_bins)
 additional_features.append(spectral_centroid)
 x = self.logmel_extractor(x)
 if self.training and self.use_spectral_cutout:
x = self.spec_augmenter(x)
 frames_num = x.shape[2]
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if len(additional_features)> 0:
additional_features.append(x)
 x = torch.cat(additional_features, dim=1)
 if self.spec_aggreagation == 'repeat3':
x = torch.cat([x,x,x], dim=1)
 elif self.spec_aggreagation == 'deltas':
delta_1 = make_delta(x)
 delta_2 = make_delta(delta_1)
 x = torch.cat([x,delta_1,delta_2], dim=1)
 elif self.spec_aggreagation == 'time_freq_encoding':
freq_encode = self._add_frequency_encoding(x)
 time_encode = self._add_time_encoding(x)
 x = torch.cat([x, freq_encode, time_encode], dim=1)
 elif self.spec_aggreagation in ['conv1','conv3']:
pass
return x, frames_num
def forward(self, input):

 x, _ = self.preprocess(input)
 x = self.encoder(x)
 if self.classifier_type == 'multiscale_relu':
logits = torch.mean(torch.stack([self.classifier(self.big_dropout(x)) for _ in range(5)],dim=0),dim=0)
 else:
logits = self.classifier(x)
 if self.use_sigmoid:
return torch.sigmoid(logits)
 else:
return logits<load_pretrained>
def load_chkp(
 path: str,
nn_model: nn.Module 
):
chckp = torch.load(path, map_location='cpu')
 nn_model.load_state_dict(chckp)
 return nn_model
def initalize_spectral_model(
 model_config: Mapping[str, Any],
chkp_path: str,
m_device: str 
):
temp_model = SpectralEffnet(
 **model_config,
device=m_device 
)
 temp_model = load_chkp(chkp_path, temp_model)
 temp_model.eval()
return temp_model<init_hyperparams>
%%time
df_train, df_test = preprocess_data()
X_train, X_test = create_features(df_train, df_test)
 y_train = df_train.loc[:, "SalePrice"]
xgb_params = {'max_depth': 4, 'learning_rate': 0.00875, 'n_estimators': 3515, 'min_child_weight': 2, 'colsample_bytree': 0.2050378195385253, 'subsample': 0.40369887914955715, 'reg_alpha': 0.3301567121037565, 'reg_lambda': 0.046181862052743}
xgb = XGBRegressor(**xgb_params, random_state=42)
 print("
X_train shape: ", X_train.shape)
 print("X_test shape: ", X_test.shape)
 print("score: ", score_dataset(X_train, y_train, xgb))
<train_model><EOS> 
df_train, df_test = preprocess_data()
X_train, X_test = create_features(df_train, df_test)
 y_train = df_train.loc[:, "SalePrice"]
xgb = XGBRegressor(**xgb_params, random_state=42)
 xgb.fit(X_train, np.log(y_train))
predictions = np.exp(xgb.predict(X_test))
output = pd.DataFrame({'Id': X_test.index, 'SalePrice': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )