Kaggle competition: Mechanisms of Action (MoA) Prediction
for col in ['BsmtFinType1','BsmtFinType2']:
df_all_red[col].replace(to_replace=['Unf', 'LwQ','Rec','BLQ','ALQ','GLQ'], value=[1/6,2/6,3/6,4/6,5/6,1], inplace=True)
 for col in ['BsmtCond', 'BsmtQual', 'GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual','HeatingQC','FireplaceQu','PoolQC']:
df_all_red[col].replace(to_replace=['NA','Po','Fa', 'TA','Gd','Ex'], value=[0,1/5,2/5,3/5,4/5,1], inplace=True)
 df_all_red["Functional"].replace(['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],[1/7,2/7,3/7,4/7,5/7,6/7,1] , inplace = True)
 df_all_red["BsmtExposure"].replace(['No','Mn','Av','Gd'],[1/4,2/4,3/4,1] , inplace = True)
 ratingCols= ['BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','BsmtExposure','GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual', \
"HeatingQC","Functional",'FireplaceQu','PoolQC']
df_all_red[ratingCols] = df_all_red[ratingCols]*100
df_all_red['BsmtScore']= df_all_red[["BsmtCond", "BsmtQual","BsmtExposure","BsmtFinType1", "BsmtFinType2"]].sum(axis = 1)/5
df_all_red['GarageScore']=df_all_red[["GarageQual", "GarageCond"]].sum(axis = 1)/2
df_all_red['ExterScore'] = df_all_red[["ExterQual", "ExterCond"]].sum(axis = 1)/2
df_all_red['QualScore'] = df_all_red[["OverallQual", "ExterQual", "GarageQual", "BsmtQual","BsmtFinType1", "BsmtFinType2", "KitchenQual","PoolQC", \
"FireplaceQu"]].sum(axis = 1)/9
df_all_red['CondScore'] = df_all_red[["OverallCond", "ExterCond", "GarageCond", "BsmtCond", "HeatingQC", "Functional"]].sum(axis = 1)/6
df_all_red['GarageGrade'] = df_all_red["GarageQual"] * df_all_red["GarageCond"]
df_all_red['ExterGrade'] = df_all_red["ExterQual"] * df_all_red["ExterCond"]
df_all_red["KitchenGrade"] = df_all_red["KitchenAbvGr"] * df_all_red["KitchenQual"]
df_all_red['BsmtGrade'] = df_all_red["BsmtCond"] * df_all_red["BsmtQual"]
df_all_red['FireplaceGrade'] = df_all_red["Fireplaces"] * df_all_red["FireplaceQu"]
df_all_red['PoolGrade'] = df_all_red["PoolArea"] * df_all_red["PoolQC"] 
<count_values>
train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_targets_scored = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train_targets_nonscored = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv")
 test_features = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 sample_submission = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv" )
for col in num_cat_Val['Variables']:
print(df_all_red[col].value_counts() )<feature_engineering>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
df_all_red["MSSubClass_1-Story"] = np.where(df_all_red.MSSubClass.isin([20,30,40,120]), 1, 0)
 df_all_red["MSSubClass_1-1/2Story"] = np.where(df_all_red.MSSubClass.isin([45,50,150]), 1, 0)
 df_all_red["MSSubClass_2-Story"] =np.where(df_all_red.MSSubClass.isin([60,70,160]), 1, 0)
 df_all_red["MSSubClass_2-1/2Story"] = np.where(df_all_red.MSSubClass.isin([75]), 1, 0)
 df_all_red["MSSubClass_Split"] = np.where(df_all_red.MSSubClass.isin([80,85]), 1, 0)
 df_all_red["MSSubClass_AllAges"] = np.where(df_all_red.MSSubClass.isin([40,45,50,75,90,150, 190]), 1, 0)
 df_all_red["MSSubClass_Newer"] = np.where(df_all_red.MSSubClass.isin([20, 60, 120, 160]), 1, 0)
 df_all_red["MSSubClass_Older"] = np.where(df_all_red.MSSubClass.isin([30, 70]), 1, 0)
 df_all_red["MSSubClass_AllStyles"] = np.where(df_all_red.MSSubClass.isin([20, 90, 190]), 1, 0)
 df_all_red["MSSubClass_Pud"] = np.where(df_all_red.MSSubClass.isin([150, 160, 180]), 1, 0)
 df_all_red["MSSubClass_SplitMultiLevel"] = np.where(df_all_red.MSSubClass.isin([80,85, 180]), 1, 0)
<drop_column>
def add_stats(df, columns, prefix):
df[prefix + '_mean'] = columns.mean(axis=1)
 df[prefix + '_sum'] = columns.sum(axis=1)
 df[prefix + '_std'] = columns.std(axis=1)
 df[prefix + '_kurt'] = columns.kurtosis(axis = 1)
 df[prefix + '_skew'] = columns.skew(axis = 1 )
df_all_red.drop(["MSSubClass"], axis = 1, inplace = True )<feature_engineering>
train_full = train_features.merge(train_targets_scored, on='sig_id')
 train_full = train_full[train_full.cp_type != 'ctl_vehicle']
train_full = train_full.drop(columns=['sig_id', 'cp_type'])
 train_full['cp_dose'] = pd.get_dummies(train_full['cp_dose'], drop_first=True)
 X_train = train_full.iloc[:, :874]
y_train = train_full.iloc[:, 874:]
X_test = test_features
X_test = X_test[X_test.cp_type != 'ctl_vehicle']
X_test = X_test.drop(columns=['sig_id', 'cp_type'])
 X_test['cp_dose'] = pd.get_dummies(X_test['cp_dose'], drop_first=True)
 add_stats(X_train, X_train.iloc[:, 2:774], 'g')
 add_stats(X_train, X_train.iloc[:, 774:874], 'c')
 add_stats(X_train, X_train.iloc[:, 2:874], 'gc')
 add_stats(X_test, X_test.iloc[:, 2:774], 'g')
 add_stats(X_test, X_test.iloc[:, 774:874], 'c')
 add_stats(X_test, X_test.iloc[:, 2:874], 'gc')
 X_full = pd.concat([X_train, X_test] )
print("Collapsing categories...")
 df_all_red['LotShape'] = np.where(df_all_1.LotShape != 'Reg', 'IR', df_all_1['LotShape'])
 df_all_red['LandContour'] = np.where(df_all_1.LandContour != 'Lvl', 'IR', df_all_1['LandContour'])
 df_all_red['RoofMatl'] = np.where(df_all_1.RoofMatl != 'CompShg', 'Non-Standard', df_all_1['RoofMatl'])
 df_all_red['Heating'] = np.where(df_all_1.Heating.str.contains("Gas"), 1, 0)
 df_all_red["MSZoning"] = np.where(df_all_red.MSZoning.isin(["RH", "RM", "RL"]), 1, 0)
 df_all_red['Electrical'] = np.where(df_all_1.Electrical != 'SBrkr', 'non-Standard', df_all_1['Electrical'])
 df_all_red['SaleCondition'] = np.where(df_all_1.SaleCondition != 'Normal', 'non-Normal', df_all_1['SaleCondition'])
 df_all_red["SaleType"] = np.where(df_all_1.SaleType.isin(["Con", "ConLD", "ConLI", "ConLw"]), "Contract", df_all_1.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["WD", "CWD"]), "Warranty", df_all_red.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["COD"]), "Oth", df_all_red.SaleType)
 df_all_red["Foundation"] = np.where(df_all_1["Foundation"] == "Stone", "BrkTil", df_all_1["Foundation"])
 df_all_red["Foundation"] = np.where(df_all_red["Foundation"] == "Wood", "PConc", df_all_red["Foundation"])
 df_all_red["BldgType"] = np.where(df_all_1.BldgType.isin(['1Fam', '2fmCon']), "Family", df_all_1.BldgType)
 df_all_red["BldgType"] = np.where(df_all_red.BldgType.isin(['TwnhsE', 'Twnhs']), "Townhouse", df_all_red.BldgType )<feature_engineering>
def pca_analyse(columns, num_comp, scaler=None):
pca = PCA(n_components=num_comp)
 if scaler is not None:
columns = scaler.fit_transform(columns)
 pca.fit(columns)
 return np.cumsum(pca.explained_variance_ratio_ )
df_all_red["CNormal"] = np.where(( df_all_1.Condition1.isin(["Norm"])) |(df_all_1.Condition2.isin(["Norm"])) , 1, 0)
 df_all_red["CRailroad"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) , 1, 0)
 df_all_red["CPark"] = np.where(( df_all_1.Condition1.isin(["PosN", "PosA"])) |(df_all_1.Condition2.isin(["PosN", "PosA"])) , 1, 0)
 df_all_red["CAdjacent"] = np.where(( df_all_1.Condition1.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) |(df_all_1.Condition2.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) , 1, 0)
 df_all_red["CWithin"] = np.where(( df_all_1.Condition1.isin(["RRNn", "PosN", "RRNe"])) |(df_all_1.Condition2.isin(["RRNn", "PosN", "RRNe"])) , 1, 0)
 df_all_red["CNorthS"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn"])) , 1, 0)
 df_all_red["CEastW"] = np.where(( df_all_1.Condition1.isin(["RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNe", "RRAe"])) , 1, 0)
 df_all_red["CStreet"] = np.where(( df_all_1.Condition1.isin(["Artery", "Feedr"])) |(df_all_1.Condition2.isin(["Artery", "Feedr"])) , 1, 0)
 df_all_red["HouseStyle_Floor"] = np.where(df_all_1.HouseStyle.isin(["1Story"]), "1", "2")
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["1.5Fin", "1.5Unf"]), "1.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["2.5Fin", "2.5Unf"]), "2.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["SLvl", "SFoyer"]), "Split", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_IsSplit"] = np.where(df_all_red.HouseStyle_Floor == 'Split',1,0)
 df_all_red['HasMasVnrType'] = np.where(df_all_1['MasVnrType'] == "None",0,1 )<drop_column>
pca_nonscaled = pca_analyse(X_full.iloc[:, 2:774], 700)
 pca_stand = pca_analyse(X_full.iloc[:, 2:774], 700, StandardScaler())
 pca_quant = pca_analyse(X_full.iloc[:, 2:774], 700, QuantileTransformer(output_distribution="normal"))
df_all_red.drop(["Condition1", "Condition2", "YrSold"], axis = 1, inplace = True )<create_dataframe>
pca_nonscaled = pca_analyse(X_full.iloc[:, 774:874], 100)
 pca_stand = pca_analyse(X_full.iloc[:, 774:874], 100, StandardScaler())
 pca_quant = pca_analyse(X_full.iloc[:, 774:874], 100, QuantileTransformer(output_distribution="normal"))
df_all_2 = df_all_red.copy()<categorify>
def add_pca(df, columns, num_comp, prefix, scaler):
pca = PCA(n_components=num_comp)
 if scaler is not None:
columns = scaler.fit_transform(columns)
 comps = pca.fit_transform(columns)
 pca_df = pd.DataFrame(comps,
columns=[prefix + f"_pca_{i}".format(i)for i in range(num_comp)])
 return pd.concat(( df.reset_index(drop=True), pca_df.reset_index(drop=True)) , axis=1)
labelencoder = preprocessing.LabelEncoder()
label_cols = [col for col in df_all_red.columns if len(df_all_red[col].value_counts())== 2]
for col in label_cols:
df_all_2[col] = labelencoder.fit_transform(df_all_red[col] )<define_variables>
def variance_threshold(df, columns, thr=0.85):
vt = VarianceThreshold(threshold=thr)
 var = vt.fit(columns ).variances_
drop_cols = columns.columns[var < thr]
return df.drop(columns=drop_cols )
cat_cols = df_all_2.columns[(df_all_2.dtypes== 'object')|(df_all_2.dtypes =='category')]
print("There are " + str(len(cat_cols)) +" categorical variables require one hot encoding.")
 cat_cols<categorify>
train_full = train_features.merge(train_targets_scored, on='sig_id')
 X_full = add_pca(X_full, X_full.iloc[:, 2:774], 600, 'g',
QuantileTransformer(output_distribution="normal"))
X_full = add_pca(X_full, X_full.iloc[:, 774:874], 80, 'c',
QuantileTransformer(output_distribution="normal"))
X_full = variance_threshold(X_full, X_full.drop(columns=['cp_time', 'cp_dose']), thr=0.9)
 X_train = X_full[:X_train.shape[0]]
X_test = X_full[-X_test.shape[0]:]
print("One hot encoding...")
 original_columns = list(df_all_2.columns)
 df_final= pd.get_dummies(df_all_2, columns=cat_cols, dummy_na=False, drop_first=True)
 new_columns = [c for c in df_final.columns if c not in original_columns]
print("There are " + str(len(new_columns)) + " variables have been created after one hot encoding" )<save_to_csv>
class MoaDataset(Dataset):
def __init__(self, X, y=None):
self.X = X
self.y = y
self.device = device
def __getitem__(self, idx):
if self.y is None:
return torch.tensor(self.X[idx], dtype=torch.float)
 return torch.tensor(self.X[idx], dtype=torch.float), \
torch.tensor(self.y[idx], dtype=torch.float)
 def __len__(self):
return self.X.shape[0]
<load_from_csv>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
<feature_engineering>
def valid_epoch(model, criterion, dataloader, device):
model.eval()
with torch.no_grad() :
total_loss = 0.
 total_num = 0
for X, y in dataloader:
X = X.to(device)
 y = y.to(device)
 outputs = model(X)
 loss = criterion(outputs, y)
 total_loss += loss.item() * X.shape[0]
total_num += X.shape[0]
return total_loss / total_num
def train_epoch(model, criterion, optimizer, scheduler, train_dataloader, valid_dataloader, device):
model.train()
total_loss = 0.
 total_num = 0
for X, y in train_dataloader:
X = X.to(device)
 y = y.to(device)
 optimizer.zero_grad()
outputs = model(X)
 loss = criterion(outputs, y)
 loss.backward()
optimizer.step()
scheduler.step()
total_loss += loss.detach().item() * X.shape[0]
total_num += X.shape[0]
train_loss = total_loss / total_num
val_loss = valid_epoch(model, criterion, val_dataloader, device)
 return train_loss, val_loss
def predict(model, dataset, device):
model.eval()
result = []
with torch.no_grad() :
for elem in dataset:
elem = elem.unsqueeze(0 ).to(device)
 output = model(elem)
 result.append(torch.sigmoid(output ).cpu().numpy())
 return np.concatenate(result )
df_final['SalePrice'] = df_all['SalePrice']
df_final['ID'] = df_all['Id']
print("The final dataset has " + str(df_final.shape[0])+ " observations with " + str(df_final.shape[1]-2)+ " explanatory variables and 1 response variable")
 df_final.head(5 )<prepare_x_and_y>
device = 'cuda'
train_loss_list = []
val_loss_list = []
pred_list = []
test_dataset = MoaDataset(X_test.values)
 num_features = X_train.shape[1]
num_targets = y_train.shape[1]
criterion = torch.nn.BCEWithLogitsLoss()
seed_list = np.arange(7)
 for seed in seed_list:
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7, shuffle=True, random_state=seed)
 for fold_ind,(train_idx, val_idx)in enumerate(mskf.split(X_train, y_train)) :
curr_train_loss_list = []
curr_val_loss_list = []
train_dataset = MoaDataset(X_train.values[train_idx], y_train.values[train_idx])
 train_dataloader = DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=1)
 val_dataset = MoaDataset(X_train.values[val_idx], y_train.values[val_idx])
 val_dataloader = DataLoader(val_dataset, batch_size=128, shuffle=True, num_workers=1)
 model = Model(num_features, num_targets, 1024 ).to(device)
 optimizer = optim.Adam(model.parameters() , lr=1e-3, weight_decay=1e-5)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=25, steps_per_epoch=len(train_dataloader))
for epoch_num in range(25):
train_loss, val_loss = train_epoch(model, criterion, optimizer, scheduler,
train_dataloader, val_dataloader, device)
 curr_train_loss_list.append(train_loss)
 curr_val_loss_list.append(val_loss)
 print('seed: ', seed, ' fold: ', fold_ind, ' epoch_num: ', epoch_num, ' train loss: ', train_loss)
 print('seed: ', seed, ' fold: ', fold_ind, ' epoch_num: ', epoch_num, ' val loss: ', val_loss)
 train_loss_list.append(curr_train_loss_list)
 val_loss_list.append(curr_val_loss_list)
 pred_list.append(predict(model, test_dataset, device))
train = df_final[df_final.ID.isin(df_train.Id)]
test = df_final[df_final.ID.isin(df_test.Id)]
X = train.drop(['SalePrice', "ID"], axis=1)
 y = np.array(train['SalePrice'])
<choose_model_class>
print("curr_train_loss: ", sum([elem[-1] for elem in train_loss_list])/ len(train_loss_list))
print("curr_val_loss: ", sum([elem[-1] for elem in val_loss_list])/ len(val_loss_list))
k_fold = KFold(n_splits=5, shuffle=True, random_state=79 )<define_search_space>
y_test = np.zeros(( sample_submission.shape[0], y_train.shape[1]))
y_test[test_features.cp_type != 'ctl_vehicle'] = sum(pred_list)/ len(pred_list)
<init_hyperparams>
submission = pd.DataFrame(y_test, columns=train_targets_scored.columns[1:])
 submission['sig_id'] = test_features['sig_id']
param_grid = {
'max_depth': [4],
'colsample_bytree': [0.4] ,
'subsample': [0.8],
'gamma': [0],
'alpha': [13],
'lambda': [1],
'learning_rate': [0.1],
'n_estimators' : np.arange(400,1000,50),
'random_state' : [79]
}
xgb_est = xgb.XGBRegressor(booster='gbtree', objective ="reg:squarederror")
 xgb_model = GridSearchCV(xgb_est,
param_grid,
scoring = 'neg_mean_squared_error',
cv = k_fold,
n_jobs = -1,
verbose = 1)
 xgb_model.fit(X,y )<find_best_params>
submission.to_csv('submission.csv', index=False )
<split><EOS> 
submission.to_csv('submission.csv', index=False )