Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<split>
warnings.filterwarnings("ignore" )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
<train_model>
!pip install -q /kaggle/input/iterative-stratification/iterative-stratification-master/
rf = RandomForestClassifier(max_depth = 10, n_estimators = 256)
 rf.fit(X_train.values, y_train.values.ravel() )<train_model>
from fastai.basics import *
y_pred = rf.predict(X_test)
 y_pred.shape
y_test.shape
print("Train Accuracy: ", accuracy_score(y_train, rf.predict(X_train)))
 print("Test Accuracy: ", accuracy_score(y_test, y_pred))<compute_test_metric>
datapath = Path("/kaggle/input/lish-moa" )
print(classification_report(y_test, y_pred))<load_from_csv>
train_features_df = pd.read_csv(datapath/'train_features.csv')
 test_features_df = pd.read_csv(datapath/'test_features.csv')
 train_targets_scored_df = pd.read_csv(datapath/'train_targets_scored.csv')
 train_targets_nonscored_df = pd.read_csv(datapath/'train_targets_nonscored.csv' )
test_data = pd.read_csv('.. /input/test.csv')
 test_data.head()<prepare_output>
[(k,train_target_distrib[k])for k in train_target_distrib if train_target_distrib[k] == 1]
test_data = preprocess_data(test_data)
 test_data.head()<drop_column>
n_train, n_test = train_features_df.shape[0], test_features_df.shape[0]
train_test_features_df = pd.concat([train_features_df, test_features_df] ).reset_index(drop=True)
 g_cols = list(o for o in train_test_features_df.columns if o.startswith("g-"))
c_cols = list(o for o in train_test_features_df.columns if o.startswith("c-"))
gc_cols = g_cols + c_cols
len(g_cols), len(c_cols), len(gc_cols )
test_data = test_data.drop('Id', 1)
 test_data = test_data.drop('Address', 1 )<predict_on_test>
from seaborn import distplot
y_pred_proba = rf.predict_proba(test_data )<create_dataframe>
ParamConfig = SimpleNamespace(
 do_rank_gauss = True,
do_quantile_tfms = False,
n_quantiles = 100,
do_pca = False,
pca_reduction_factor = 3,
do_umap = False,
umap_reduction_factor = 3,
umap_n_neighbors = 150,
do_knn_encoding = True,
knn_k = 100,
do_transfer_learning = False,
tl_n_epochs = 30,
tl_smoothing = 0.001,
tl_lr = 0.001,
ft_n_epochs = 30,
ft_smoothing = 0.001,
ft_lr = 0.001,
model_n_layers = 2,
model_layer_width = 512,
model_ps = 0.25
)
colmn = ["ARSON","ASSAULT","BAD CHECKS","BRIBERY","BURGLARY","DISORDERLY CONDUCT","DRIVING UNDER THE INFLUENCE","DRUG/NARCOTIC","DRUNKENNESS","EMBEZZLEMENT","EXTORTION","FAMILY OFFENSES","FORGERY/COUNTERFEITING","FRAUD","GAMBLING","KIDNAPPING","LARCENY/THEFT","LIQUOR LAWS","LOITERING","MISSING PERSON","NON-CRIMINAL","OTHER OFFENSES","PORNOGRAPHY/OBSCENE MAT","PROSTITUTION","RECOVERED VEHICLE","ROBBERY","RUNAWAY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUICIDE","SUSPICIOUS OCC","TREA","TRESPASS","VANDALISM","VEHICLE THEFT","WARRANTS","WEAPON LAWS"]
result = pd.DataFrame(y_pred_proba, columns=colmn)
 result.to_csv(path_or_buf="rf_predict.csv",index=True, index_label = 'Id' )<import_modules>
import cupy as cp
from cupyx.scipy.special import erfinv
import cudf as gd
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.special import erfinv as sp_erfinv
<set_options>
def to_rankgauss(x):
"https://medium.com/rapids-ai/gauss-rank-transformation-is-100x-faster-with-rapids-and-cupy-7c947e3397da"
x_cpu = x
r_cpu = x_cpu.argsort().argsort()
epsilon = 1e-6
r_cpu =(r_cpu/r_cpu.max() -0.5)*2
r_cpu = cp.clip(r_cpu,-1+epsilon,1-epsilon)
 r_cpu = sp_erfinv(r_cpu)
 return r_cpu
%matplotlib inline 
<load_from_csv>
if ParamConfig.do_rank_gauss:
for col in gc_cols:
train_test_features_df[col] = to_rankgauss(train_test_features_df[col] )
df_train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 print("There are %0.0f" %df_train.shape[0] + " houses and %0.0f" %df_train.shape[1] +" variables in the training dataset")
 print("There are %0.0f" %df_test.shape[0] + " houses and %0.0f" %df_test.shape[1] +" explanatory variables in the test dataset")
 df_all = df_train.append(df_test)
 df_all.reset_index(inplace=True, drop=True)
 df_all.head()<drop_column>
from sklearn.preprocessing import QuantileTransformer
df_all_1 = df_all.copy()
df_all_1 = df_all.drop(['Id','SalePrice'], axis = 1 )<feature_engineering>
if ParamConfig.do_quantile_tfms:
transformer = QuantileTransformer(n_quantiles=ParamConfig.n_quantiles, random_state=0, output_distribution="normal")
 train_test_features_df[gc_cols] = transformer.fit_transform(train_test_features_df[gc_cols] )
df_all_1['MissingCount'] = df_all_1.isnull().sum(axis =1)
 df_all_1['MiscFeature'] = np.where(( df_all.MiscFeature.isnull() == True), 1,0)
 df_all_1['Alley'] = np.where(( df_all.Alley.isnull() == True), 1,0)
 df_all_1['Fence'] = np.where(( df_all.Fence.isnull() == True), 1,0)
 print("There are %0.0f" %df_all_1.shape[1] +" explanatory variables left after removing varibles with more than 50 % missing data" )<drop_column>
train_test_features_df.groupby(['cp_type', 'cp_time', 'cp_dose'])[['sig_id']].count()
remainMiss = missingDesc.drop(['MiscFeature','Alley','Fence','SalePrice'], axis =0)
 df_all_1[remainMiss.index].dtypes<feature_engineering>
len(train_test_features_df['sig_id'].unique() )
PoolIndex = df_all_1[df_all_1['PoolArea']!=0].index
df_all_1.loc[np.logical_not(df_all_1.index.isin(PoolIndex)) ,'PoolQC'] = 'NA'
df_all_1.loc[PoolIndex,'PoolQC'].isnull().sum()
df_all_1['PoolQC'] = df_all_1['PoolQC'].fillna(df_all_1.PoolQC[PoolIndex].mode() [0])
 df_all_1.PoolQC.value_counts()<feature_engineering>
all(train_test_features_df.merge(train_targets_scored_df, on='sig_id' ).query("cp_type == 'ctl_vehicle'")[scored_cols].sum() ==0 )
FireplaceIndex = df_all_1[df_all_1['Fireplaces']==0].index
df_all_1.loc[df_all_1.index.isin(FireplaceIndex),'FireplaceQu'] = 'NA'
df_all_1.loc[FireplaceIndex,'PoolQC'].isnull().sum() 
<count_unique_values>
all(train_test_features_df.merge(train_targets_nonscored_df, on='sig_id' ).query("cp_type == 'ctl_vehicle'")[nonscored_cols].sum() ==0 )
cat_length = 30
print('Imputing non-numeric missing variables...')
 df_all_1 = df_all_1.apply(lambda x: x.fillna(x.mode() [0])if(x.dtype == "object" and len(x.unique())<= cat_length)else x, axis=0 )<sort_values>
from cuml.decomposition import PCA as cumlPCA
pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )<count_values>
%%time
if ParamConfig.do_pca:
pca_feats = []
colnames = []
for name, cols in [("gene", g_cols),("cell", c_cols)]:
n_comp = int(len(cols)/ParamConfig.pca_reduction_factor)
 pca = cumlPCA(n_components=n_comp, iterated_power=500)
 pca_feat = pca.fit_transform(train_test_features_df[cols])
 pca_feats += [pca_feat]
colnames += [f"pca_{name}_{i}" for i in range(n_comp)]
pca_feats = np.hstack(pca_feats)
 train_test_features_df[colnames] = pca_feats
for col in remainMiss.index:
if(df_all_1[col].dtypes != 'object' and len(np.unique(df_all_1[col])) < cat_length):
print(df_all_1[col].value_counts() )<feature_engineering>
from cuml.manifold import UMAP as cumlUMAP
print('Imputing 3 missing numeric variables as categorical varaibles...')
 for col in ['BsmtHalfBath', 'BsmtFullBath','GarageCars']:
df_all_1[col] = df_all_1[col].fillna(df_all_1[col].mode() [0])
 pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )<feature_engineering>
%%time
if ParamConfig.do_umap:
umap_feats = []
colnames = []
for name, cols in [("gene", g_cols),("cell", c_cols)]:
n_comp = int(len(cols)/ParamConfig.umap_reduction_factor)
 umap = cumlUMAP(n_components=n_comp, n_neighbors=ParamConfig.umap_n_neighbors)
 umap_feat = umap.fit_transform(train_test_features_df[cols])
 umap_feats += [umap_feat]
colnames += [f"umap_{name}_{i}" for i in range(n_comp)]
umap_feats = np.hstack(umap_feats)
 train_test_features_df[colnames] = umap_feats
print("Imputing GarageYrBlt...")
 temp = df_all_1[df_all_1.GarageYrBlt.isnull() == True].index
df_all_1["GarageYrBlt"]= np.where(( df_all_1.GarageYrBlt.isnull() == True), df_all_1.YearBuilt,df_all_1.GarageYrBlt ).astype('int64')
 print('Chekcing if GarageYrBlt is corretly imputed...')
(df_all_1.loc[temp,"GarageYrBlt"] == df_all_1.loc[temp,"YearBuilt"] ).unique()<create_dataframe>
for ohe_col in ['cp_type', 'cp_time', 'cp_dose']:
vals = train_test_features_df[ohe_col].unique()
if len(vals)== 2:
col = ohe_col+"s"
cat2code = {v:k for k,v in enumerate(vals)}
train_test_features_df[col] = train_test_features_df[ohe_col].map(cat2code)
 else:
cat2code = {v:k for k,v in enumerate(vals)}
encoded = train_test_features_df[ohe_col].map(cat2code)
 ohe_arr = np.zeros(( len(vals), len(vals)))
 ohe_arr[np.diag_indices_from(ohe_arr)] = 1
col = [f"{ohe_col}s_{o}" for o in vals]
train_test_features_df[col] = ohe_arr[encoded]
imputeCol = []
for col in df_all_1.columns:
if(df_all_1[col].dtypes != 'object' and len(df_all_1[col].unique())>= cat_length):
imputeCol.append(col)
 print("Columns used for KNN imputation are:
" + str(imputeCol))
imputer = KNNImputer(n_neighbors=5, weights='uniform', metric='nan_euclidean')
 X = df_all_1[imputeCol]
print('Imputing remaining numerical variables with KNN...')
 Xtrans = imputer.fit_transform(X)
 XT = pd.DataFrame(data=Xtrans)
 XT.columns = imputeCol
df_all_1[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']] \
= XT[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']]
print('There are now ' + str(df_all_1.isnull().any().sum())+ ' missing values in our data set.' )<sort_values>
train_test_features_df = train_test_features_df.drop("cp_types", 1 )
numVal =(( df_all_1.dtypes != 'object' ).index)[df_all_1.dtypes != 'object']
length = []
for col in numVal:
length.append(len(df_all_1[col].unique()))
numVal_df = pd.DataFrame({'Variables':numVal,'UniqueValuesLength':length})
 num_cat_Val = numVal_df[numVal_df['UniqueValuesLength'] < cat_length].sort_values(by='UniqueValuesLength', ascending=False)
 num_cat_Val.head()<filter>
from cuml.neighbors import KNeighborsRegressor as cumlKNN
print('There are %i' %numVal_df[numVal_df['UniqueValuesLength'] < cat_length].shape[0] + ' numerical variables and %i' \
%(df_all_1.dtypes == 'object' ).sum() + ' non-numerical variables can potentially be treated as categorcial variables with some exceptions.' )<sort_values>
if ParamConfig.do_knn_encoding:
knn = cumlKNN(n_neighbors=ParamConfig.knn_k)
 merged_nonscored_train_test_df = train_test_features_df.merge(train_targets_nonscored_df, on='sig_id', how='left')
 unique_cp_time, unique_cp_dose = merged_nonscored_train_test_df['cp_time'].unique() , merged_nonscored_train_test_df['cp_dose'].unique()
cont_val = numVal_df[numVal_df['UniqueValuesLength'] >= cat_length].sort_values(by='UniqueValuesLength')
 cont_val.head()<sort_values>
if ParamConfig.do_knn_encoding:
knn_groups = []
for i in unique_cp_time:
for j in unique_cp_dose: knn_groups.append(( i,j))
knn_cols = [f"knn{ParamConfig.knn_k}_{i}" for i in range(len(nonscored_cols)) ]
for c in knn_cols: train_test_features_df[c] = 0
yCor = cor['SalePrice'].sort_values(ascending = False ).reset_index().drop(index=0)
 yCor.columns = ["Variables", "Cor"]
yCor[(yCor["Cor"] > 0.5)|(yCor["Cor"] < -0.5)]<create_dataframe>
if ParamConfig.do_knn_encoding:
for time, dose in knn_groups:
X = merged_nonscored_train_test_df.query(f"cp_time == '{time}' & cp_dose == '{dose}'")[gc_cols]
y = merged_nonscored_train_test_df.query(f"cp_time == '{time}' & cp_dose == '{dose}'")[nonscored_cols]
idxs = array(list(X.index))
train_idxs = idxs[np.where(( y.isna().sum(1)== 0)) [0]]
test_idxs = idxs[np.where(( y.isna().sum(1)!= 0)) [0]]
X_train, y_train, X_test, y_test = X.loc[train_idxs], y.loc[train_idxs], X.loc[test_idxs], y.loc[test_idxs]
knn.fit(X_train, y_train)
 train_preds, test_preds = knn.predict(X_train), knn.predict(X_test)
 train_test_features_df.loc[train_idxs, knn_cols] = train_preds
train_test_features_df.loc[test_idxs, knn_cols] = test_preds
df_all_red = df_all_1.copy()<feature_engineering>
train_features_df = train_test_features_df[:n_train].reset_index(drop=True)
 test_features_df = train_test_features_df[n_train:].reset_index(drop=True)
 assert np.sum(train_features_df.isna() ).sum() == 0
assert np.sum(test_features_df.isna() ).sum() == 0
df_all_red.YearRemodAdd = np.where(df_all_1.YearBuilt >df_all_1.YearRemodAdd, df_all_1.YearBuilt, df_all_1.YearRemodAdd)
 df_all_red.YrSold = np.where(df_all_1.YearBuilt >df_all_1.YrSold , df_all_1.YearBuilt, df_all_1.YrSold)
 df_all_red.YrSold = np.where(df_all_red.YearRemodAdd > df_all_red.YrSold, df_all_1.YearRemodAdd, df_all_red.YrSold)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt < df_all_red.YearBuilt, df_all_red.YearBuilt, df_all_red.GarageYrBlt)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt >2010, df_all_red.YearBuilt,df_all_red.GarageYrBlt)
 df_all_red['RestorePeriod'] = df_all_red.YearRemodAdd - df_all_red.YearBuilt
df_all_red['HouseAge'] = df_all_red.YrSold - df_all_red.YearBuilt
df_all_red["RemodAge"] = df_all_red.YrSold - df_all_red.YearRemodAdd
df_all_red['GarageAge'] = df_all_red.YrSold - df_all_red.GarageYrBlt
df_all_red["IsNewHouse"] = np.where(df_all_red.YearBuilt == df_all_red.YrSold, 1 ,0)
 df_all_red["Remodeled"] = np.where(df_all_red.RestorePeriod != 0, 1 ,0)
 df_all_red["GreatDepression"] = np.where(( df_all_1.YearBuilt >=1929)&(df_all_1.YearBuilt <= 1939), 1,0)
 df_all_red["Pre-GreatDepression"] = np.where(( df_all_1.YearBuilt <1929), 1,0)
 df_all_red["MortgageCrisis"] = np.where(( df_all_1.YearBuilt >=2007)&(df_all_1.YearBuilt <= 2010), 1,0)
 df_all_red["MortgageCrisisRestoration"] = np.where(( df_all_1.YearRemodAdd >=2007)&(df_all_1.YearRemodAdd <= 2010), 1,0)
<drop_column>
from fastai.tabular.all import *
df_all_red.drop(["YearBuilt","GarageYrBlt", "YearRemodAdd"], axis = 1, inplace = True )<feature_engineering>
@log_args
class LabelSmoothingBCEWithLogits(Module):
y_int = True
def __init__(self, eps:float=0.1): store_attr('eps')
 def forward(self, output, target):
target = target + self.eps*(1-2*target)
 return F.binary_cross_entropy_with_logits(output, target)
 def decodes(self, x): return x>self.thresh
def activation(self, x): return torch.sigmoid(x )
df_all_red["TotalFlrSF"] = df_all_red["1stFlrSF"] + df_all_red["2ndFlrSF"]
df_all_red["TotalBsmtFin"] = df_all_red['BsmtFinSF1']+df_all_red['BsmtFinSF2']
df_all_red["BsmtFinRatio"] =(df_all_red['TotalBsmtFin'] / df_all_red['TotalBsmtSF'] ).fillna(0)
 df_all_red["IsBasement"] = np.where(df_all_red['TotalBsmtSF'] < 1 , 0, 1)
 df_all_red["PorchArea"] = df_all_red['OpenPorchSF'] + df_all_red['EnclosedPorch'] + df_all_red['ScreenPorch'] + \
df_all_red["3SsnPorch"] + df_all_red['WoodDeckSF']
df_all_red["TotalHouseArea"] = df_all_red['TotalFlrSF'] + df_all_red['TotalBsmtSF']
df_all_red["TotalLivArea"] = df_all_red['GrLivArea'] + df_all_red['TotalBsmtSF']
df_all_red["2ndFloor"] = np.where(( df_all_red['2ndFlrSF'] < 1), 0, 1)
 df_all_red["GarageAreaPerCar"] =(df_all_red.GarageArea / df_all_red.GarageCars ).fillna(0)
 df_all_red["HouseLotRatio"] = df_all_red.TotalHouseArea /df_all_red.LotArea
df_all_red["LivLotRatio"] = df_all_red.GrLivArea / df_all_red.LotArea
df_all_red["GarageLotRatio"] = df_all_red.GarageArea / df_all_red.LotArea
df_all_red["MasVnrRatio"] = df_all_red.MasVnrArea / df_all_red.TotalHouseArea
df_all_red["DifArea"] =(df_all_red.LotArea - df_all_red["1stFlrSF"] - df_all_red.GarageArea - df_all_red.PorchArea)
 df_all_red["LowQualFinSFRatio"] = df_all_red.LowQualFinSF/df_all_red.TotalHouseArea<count_values>
def clipped_bce(inp, targ): return F.binary_cross_entropy(torch.clamp(inp.sigmoid() ,1e-15, 1-1e-15), targ.float())
 metric = AvgMetric(clipped_bce )
for col in ['TotRmsAbvGrd', 'OverallQual', 'OverallCond','BedroomAbvGr','GarageCars','KitchenAbvGr','FullBath','MissingCount']:
print(col + ' has ' + str(len(df_all_red[col].value_counts())) + ' unique numerical values.' )<feature_engineering>
cat_names = []
cont_names = L(list(train_features_df.columns[4:]))
y_names = nonscored_cols
for col in ['OverallQual','OverallCond']:
df_all_red[col].replace(np.linspace(1,10,10 ).astype(int), list(np.linspace(10,100,10)) , inplace=True)
 df_all_red['OverallScore'] = df_all_red[['OverallQual', 'OverallCond']].sum(axis = 1)/2
df_all_red["OverallGrade"] = df_all_red["OverallQual"] * df_all_red["OverallCond"]<categorify>
merged_nonscored_train_df = train_features_df.merge(train_targets_nonscored_df, on='sig_id', how='left' )
for col in ['BsmtFinType1','BsmtFinType2']:
df_all_red[col].replace(to_replace=['Unf', 'LwQ','Rec','BLQ','ALQ','GLQ'], value=[1/6,2/6,3/6,4/6,5/6,1], inplace=True)
 for col in ['BsmtCond', 'BsmtQual', 'GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual','HeatingQC','FireplaceQu','PoolQC']:
df_all_red[col].replace(to_replace=['NA','Po','Fa', 'TA','Gd','Ex'], value=[0,1/5,2/5,3/5,4/5,1], inplace=True)
 df_all_red["Functional"].replace(['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],[1/7,2/7,3/7,4/7,5/7,6/7,1] , inplace = True)
 df_all_red["BsmtExposure"].replace(['No','Mn','Av','Gd'],[1/4,2/4,3/4,1] , inplace = True)
 ratingCols= ['BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','BsmtExposure','GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual', \
"HeatingQC","Functional",'FireplaceQu','PoolQC']
df_all_red[ratingCols] = df_all_red[ratingCols]*100
df_all_red['BsmtScore']= df_all_red[["BsmtCond", "BsmtQual","BsmtExposure","BsmtFinType1", "BsmtFinType2"]].sum(axis = 1)/5
df_all_red['GarageScore']=df_all_red[["GarageQual", "GarageCond"]].sum(axis = 1)/2
df_all_red['ExterScore'] = df_all_red[["ExterQual", "ExterCond"]].sum(axis = 1)/2
df_all_red['QualScore'] = df_all_red[["OverallQual", "ExterQual", "GarageQual", "BsmtQual","BsmtFinType1", "BsmtFinType2", "KitchenQual","PoolQC", \
"FireplaceQu"]].sum(axis = 1)/9
df_all_red['CondScore'] = df_all_red[["OverallCond", "ExterCond", "GarageCond", "BsmtCond", "HeatingQC", "Functional"]].sum(axis = 1)/6
df_all_red['GarageGrade'] = df_all_red["GarageQual"] * df_all_red["GarageCond"]
df_all_red['ExterGrade'] = df_all_red["ExterQual"] * df_all_red["ExterCond"]
df_all_red["KitchenGrade"] = df_all_red["KitchenAbvGr"] * df_all_red["KitchenQual"]
df_all_red['BsmtGrade'] = df_all_red["BsmtCond"] * df_all_red["BsmtQual"]
df_all_red['FireplaceGrade'] = df_all_red["Fireplaces"] * df_all_red["FireplaceQu"]
df_all_red['PoolGrade'] = df_all_red["PoolArea"] * df_all_red["PoolQC"] 
<count_values>
ctl_sig_ids = merged_nonscored_train_df.query("cp_type=='ctl_vehicle'")['sig_id'].values
trn_df = merged_nonscored_train_df[~merged_nonscored_train_df['sig_id'].isin(ctl_sig_ids)]
for col in num_cat_Val['Variables']:
print(df_all_red[col].value_counts() )<feature_engineering>
procs = []
dls = TabularDataLoaders.from_df(merged_nonscored_train_df,
procs=procs,
cat_names=cat_names,
cont_names=cont_names,
y_names=y_names,
valid_idx=[],
bs=128 )
df_all_red["MSSubClass_1-Story"] = np.where(df_all_red.MSSubClass.isin([20,30,40,120]), 1, 0)
 df_all_red["MSSubClass_1-1/2Story"] = np.where(df_all_red.MSSubClass.isin([45,50,150]), 1, 0)
 df_all_red["MSSubClass_2-Story"] =np.where(df_all_red.MSSubClass.isin([60,70,160]), 1, 0)
 df_all_red["MSSubClass_2-1/2Story"] = np.where(df_all_red.MSSubClass.isin([75]), 1, 0)
 df_all_red["MSSubClass_Split"] = np.where(df_all_red.MSSubClass.isin([80,85]), 1, 0)
 df_all_red["MSSubClass_AllAges"] = np.where(df_all_red.MSSubClass.isin([40,45,50,75,90,150, 190]), 1, 0)
 df_all_red["MSSubClass_Newer"] = np.where(df_all_red.MSSubClass.isin([20, 60, 120, 160]), 1, 0)
 df_all_red["MSSubClass_Older"] = np.where(df_all_red.MSSubClass.isin([30, 70]), 1, 0)
 df_all_red["MSSubClass_AllStyles"] = np.where(df_all_red.MSSubClass.isin([20, 90, 190]), 1, 0)
 df_all_red["MSSubClass_Pud"] = np.where(df_all_red.MSSubClass.isin([150, 160, 180]), 1, 0)
 df_all_red["MSSubClass_SplitMultiLevel"] = np.where(df_all_red.MSSubClass.isin([80,85, 180]), 1, 0)
<drop_column>
tl_learner = tabular_learner(dls,
layers=ParamConfig.model_n_layers*[ParamConfig.model_layer_width],
config = {"ps": ParamConfig.model_n_layers*[ParamConfig.model_ps]},
n_out=len(nonscored_cols),
loss_func=LabelSmoothingBCEWithLogits(ParamConfig.tl_smoothing),
metrics=[metric] )
df_all_red.drop(["MSSubClass"], axis = 1, inplace = True )<feature_engineering>
tl_learner.model
print("Collapsing categories...")
 df_all_red['LotShape'] = np.where(df_all_1.LotShape != 'Reg', 'IR', df_all_1['LotShape'])
 df_all_red['LandContour'] = np.where(df_all_1.LandContour != 'Lvl', 'IR', df_all_1['LandContour'])
 df_all_red['RoofMatl'] = np.where(df_all_1.RoofMatl != 'CompShg', 'Non-Standard', df_all_1['RoofMatl'])
 df_all_red['Heating'] = np.where(df_all_1.Heating.str.contains("Gas"), 1, 0)
 df_all_red["MSZoning"] = np.where(df_all_red.MSZoning.isin(["RH", "RM", "RL"]), 1, 0)
 df_all_red['Electrical'] = np.where(df_all_1.Electrical != 'SBrkr', 'non-Standard', df_all_1['Electrical'])
 df_all_red['SaleCondition'] = np.where(df_all_1.SaleCondition != 'Normal', 'non-Normal', df_all_1['SaleCondition'])
 df_all_red["SaleType"] = np.where(df_all_1.SaleType.isin(["Con", "ConLD", "ConLI", "ConLw"]), "Contract", df_all_1.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["WD", "CWD"]), "Warranty", df_all_red.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["COD"]), "Oth", df_all_red.SaleType)
 df_all_red["Foundation"] = np.where(df_all_1["Foundation"] == "Stone", "BrkTil", df_all_1["Foundation"])
 df_all_red["Foundation"] = np.where(df_all_red["Foundation"] == "Wood", "PConc", df_all_red["Foundation"])
 df_all_red["BldgType"] = np.where(df_all_1.BldgType.isin(['1Fam', '2fmCon']), "Family", df_all_1.BldgType)
 df_all_red["BldgType"] = np.where(df_all_red.BldgType.isin(['TwnhsE', 'Twnhs']), "Townhouse", df_all_red.BldgType )<feature_engineering>
if ParamConfig.do_transfer_learning: tl_learner.fit_flat_cos(ParamConfig.tl_n_epochs, lr=ParamConfig.tl_lr )
df_all_red["CNormal"] = np.where(( df_all_1.Condition1.isin(["Norm"])) |(df_all_1.Condition2.isin(["Norm"])) , 1, 0)
 df_all_red["CRailroad"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) , 1, 0)
 df_all_red["CPark"] = np.where(( df_all_1.Condition1.isin(["PosN", "PosA"])) |(df_all_1.Condition2.isin(["PosN", "PosA"])) , 1, 0)
 df_all_red["CAdjacent"] = np.where(( df_all_1.Condition1.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) |(df_all_1.Condition2.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) , 1, 0)
 df_all_red["CWithin"] = np.where(( df_all_1.Condition1.isin(["RRNn", "PosN", "RRNe"])) |(df_all_1.Condition2.isin(["RRNn", "PosN", "RRNe"])) , 1, 0)
 df_all_red["CNorthS"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn"])) , 1, 0)
 df_all_red["CEastW"] = np.where(( df_all_1.Condition1.isin(["RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNe", "RRAe"])) , 1, 0)
 df_all_red["CStreet"] = np.where(( df_all_1.Condition1.isin(["Artery", "Feedr"])) |(df_all_1.Condition2.isin(["Artery", "Feedr"])) , 1, 0)
 df_all_red["HouseStyle_Floor"] = np.where(df_all_1.HouseStyle.isin(["1Story"]), "1", "2")
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["1.5Fin", "1.5Unf"]), "1.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["2.5Fin", "2.5Unf"]), "2.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["SLvl", "SFoyer"]), "Split", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_IsSplit"] = np.where(df_all_red.HouseStyle_Floor == 'Split',1,0)
 df_all_red['HasMasVnrType'] = np.where(df_all_1['MasVnrType'] == "None",0,1 )<drop_column>
if ParamConfig.do_transfer_learning: tl_learner.save("pretrained_tabular", with_opt=False);
df_all_red.drop(["Condition1", "Condition2", "YrSold"], axis = 1, inplace = True )<create_dataframe>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
df_all_2 = df_all_red.copy()<categorify>
merged_scored_train_df = train_features_df.merge(train_targets_scored_df, on='sig_id', how='left')
 ctl_sig_ids = merged_scored_train_df.query("cp_type=='ctl_vehicle'")['sig_id'].values
trn_df = merged_scored_train_df[~merged_scored_train_df['sig_id'].isin(ctl_sig_ids)]
labelencoder = preprocessing.LabelEncoder()
label_cols = [col for col in df_all_red.columns if len(df_all_red[col].value_counts())== 2]
for col in label_cols:
df_all_2[col] = labelencoder.fit_transform(df_all_red[col] )<define_variables>
y_names = scored_cols
cat_cols = df_all_2.columns[(df_all_2.dtypes== 'object')|(df_all_2.dtypes =='category')]
print("There are " + str(len(cat_cols)) +" categorical variables require one hot encoding.")
 cat_cols<categorify>
N_FOLDS = 10
mskf = MultilabelStratifiedKFold(n_splits=N_FOLDS )
print("One hot encoding...")
 original_columns = list(df_all_2.columns)
 df_final= pd.get_dummies(df_all_2, columns=cat_cols, dummy_na=False, drop_first=True)
 new_columns = [c for c in df_final.columns if c not in original_columns]
print("There are " + str(len(new_columns)) + " variables have been created after one hot encoding" )<save_to_csv>
sig_ids = trn_df['sig_id'].values
cv_idxs = list(mskf.split(sig_ids, y=trn_df[y_names]))
<load_from_csv>
cvpath = Path("cv_sig_ids")
 if not cvpath.exists() : cvpath.mkdir()
for i, idxs in enumerate(cv_idxs): pd.to_pickle(sig_ids[idxs[1]], cvpath/f'sig_ids_fold{i}.pkl' )
<feature_engineering>
if ParamConfig.do_transfer_learning: pretrained_statedict = torch.load("models/pretrained_tabular.pth", map_location=default_device() )
df_final['SalePrice'] = df_all['SalePrice']
df_final['ID'] = df_all['Id']
print("The final dataset has " + str(df_final.shape[0])+ " observations with " + str(df_final.shape[1]-2)+ " explanatory variables and 1 response variable")
 df_final.head(5 )<split>
def load_top_layers(learner, pretrained_statedict):
i = 0
for n, p in learner.model.named_parameters() :
try:
p.data.copy_(pretrained_statedict[n])
 print(f"Loaded {n}")
 i += 1
except:
continue
if i == 0:
print("No parameter is loaded")
 if ParamConfig.do_transfer_learning: raise Exception("Transfer learning is set True but no parameter loaded!")
 else:
print(f"Total of loaded params: {i}" )
train = df_final[df_final.ID.isin(df_train.Id)]
test = df_final[df_final.ID.isin(df_test.Id)]
X = train.drop(['SalePrice', "ID"], axis=1)
 y = np.array(train['SalePrice'])
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=79 )<define_search_space>
for FOLD in range(N_FOLDS):
valid_sig_ids = pd.read_pickle(cvpath/f'sig_ids_fold{FOLD}.pkl')
 valid_idxs = np.where(trn_df.sig_id.isin(valid_sig_ids)) [0]
procs = []
dls = TabularDataLoaders.from_df(trn_df,
procs=procs,
cat_names=cat_names,
cont_names=cont_names,
y_names=y_names,
valid_idx=valid_idxs,
bs=128)
 learner = tabular_learner(dls,
layers=ParamConfig.model_n_layers*[ParamConfig.model_layer_width],
config = {"ps": ParamConfig.model_n_layers*[ParamConfig.model_ps]},
n_out=len(scored_cols),
loss_func=LabelSmoothingBCEWithLogits(ParamConfig.ft_smoothing),
metrics=[metric])
 if ParamConfig.do_transfer_learning: load_top_layers(learner, pretrained_statedict)
 learner.fit_flat_cos(ParamConfig.ft_n_epochs,
lr=ParamConfig.ft_lr,
cbs=[SaveModelCallback(monitor='clipped_bce',
fname=f'tabular_fold{FOLD}',
comp=np.less),
EarlyStoppingCallback(monitor='clipped_bce',
patience=5,
comp=np.less),
CSVLogger(fname=f"logs/tabular_logs_fold{FOLD}.csv")])
 learner.export(f'models/tabular_fold{FOLD}_export.pkl' )
k_fold = KFold(n_splits=5, shuffle=True, random_state=79)
 param_grid = {
'feature_fraction': np.arange(0.1,0.3,0.02),
'subsample': np.arange(0.70,0.8,0.01),
'learning_rate': [0.01],
'max_depth': [3,5,7,9],
'num_leaves' : np.arange(30,60,10),
'n_estimators' : [3000,4000,5000,7000],
'max_bin': [30,40,50,60],
"lambda_l1":np.arange(0.0, 1.1, 0.2),
"lambda_l2":np.arange(0.0, 1.1, 0.2),
"min_gain_to_split": np.arange(0, 36, 2),
}
print('Parameter tuning...')
 lgb_est = lgb.LGBMRegressor(random_state = 79, objective = "regression",
metric = "rmse",boosting_type ='gbdt')
 lgb_model = RandomizedSearchCV(lgb_est,param_grid, random_state = 79, n_jobs=-1,
cv = k_fold, scoring = "neg_mean_squared_error")
 lgb_model.fit(X_train, y_train,
eval_set = [(X_train, y_train),(X_test, y_test)],
eval_metric = ['rmse'],
eval_names = ["Train","Validation"],
early_stopping_rounds = 10,
verbose = 100 )<find_best_params>
fold_metrics = [pd.read_csv(logspath/f'tabular_logs_fold{FOLD}.csv')['clipped_bce'].min() for FOLD in range(10)]
print(lgb_model.best_params_)
 final_model = lgb_model.best_estimator_<train_model>
modelspath = Path("models" )
i = 0
train_pred = np.zeros(X.shape[0])
 test_pred = []
fold = ['1st', '2nd', '3rd', '4th', '5th']
for train_index, test_index in k_fold.split(X):
print("TRAIN:", train_index[:10], "TEST:", test_index[:10])
 X_train, X_test = X.loc[train_index,:], X.loc[test_index,:]
y_train, y_test = y[train_index], y[test_index]
cv_model = lgb.LGBMRegressor(random_state=79, objective = "regression",
metric = "rmse",**lgb_model.best_params_)
 cv_model.fit(X_train,y_train)
 print("Making prediction for the " + fold[i] + " fold...")
 train_pred[test_index] = cv_model.predict(X_test)
 test_pred.append(cv_model.predict(test.drop(["ID","SalePrice"], axis = 1)))
 i += 1<compute_test_metric>
fold_preds = []
for i in range(10):
learner = load_learner(modelspath/f'tabular_fold{i}_export.pkl')
 test_dl = learner.dls.test_dl(test_features_df)
 preds, _ = learner.get_preds(dl=test_dl)
 fold_preds += [preds]
preds = torch.stack(fold_preds )
cv_score = np.mean(cross_val_score(final_model,X,y,scoring = 'neg_root_mean_squared_error', cv=k_fold,n_jobs = -1,verbose =0))
print('RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred)) ,2))
print('Cross Validation RMSE for training set :', round(-cv_score,2))<define_variables>
mean_preds = preds.mean(0 )
final_pred = sum(test_pred)/5<save_to_csv>
sub_df = pd.DataFrame(mean_preds, columns=test_dl.y_names)
 sub_df['sig_id'] = test_dl.items['sig_id']
sub_df = sub_df[['sig_id']+test_dl.y_names]
submission = pd.DataFrame({"Id": test.ID, "SalePrice": final_pred})
 submission.to_csv("submission.csv", index = None)
 submission<import_modules>
ctl_sig_ids = test_features_df.query("cp_type == 'ctl_vehicle'")['sig_id'].values
<set_options>
sub_df.loc[sub_df.sig_id.isin(ctl_sig_ids), test_dl.y_names] = 0
%matplotlib inline 
<load_from_csv>
sub_df.to_csv("submission.csv", index=False )