Kaggle competition: Mechanisms of Action (MoA) Prediction
df_train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 print("There are %0.0f" %df_train.shape[0] + " houses and %0.0f" %df_train.shape[1] +" variables in the training dataset")
 print("There are %0.0f" %df_test.shape[0] + " houses and %0.0f" %df_test.shape[1] +" explanatory variables in the test dataset")
 df_all = df_train.append(df_test)
 df_all.reset_index(inplace=True, drop=True)
 df_all.head()<drop_column>
dtype = torch.float32
device = torch.device('cuda:0')
 seed = 123456
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=seed )
df_all_1 = df_all.copy()
df_all_1 = df_all.drop(['Id','SalePrice'], axis = 1 )<feature_engineering>
data_path = './data/' if 'windows' in platform.platform().lower() else '.. /input/lish-moa/'
df_all_1['MissingCount'] = df_all_1.isnull().sum(axis =1)
 df_all_1['MiscFeature'] = np.where(( df_all.MiscFeature.isnull() == True), 1,0)
 df_all_1['Alley'] = np.where(( df_all.Alley.isnull() == True), 1,0)
 df_all_1['Fence'] = np.where(( df_all.Fence.isnull() == True), 1,0)
 print("There are %0.0f" %df_all_1.shape[1] +" explanatory variables left after removing varibles with more than 50 % missing data" )<drop_column>
train_features = pd.read_csv(data_path + 'train_features.csv')
 test_features = pd.read_csv(data_path + 'test_features.csv')
 train_targets_scored = pd.read_csv(data_path + 'train_targets_scored.csv' )
remainMiss = missingDesc.drop(['MiscFeature','Alley','Fence','SalePrice'], axis =0)
 df_all_1[remainMiss.index].dtypes<feature_engineering>
class MoADataset(Dataset):
def __init__(self, dtype, features, targets=None, feature_columns=None):
self.dtype = dtype
if isinstance(features,(pd.DataFrame, pd.Series)) :
if feature_columns is not None:
features = features[feature_columns]
features = features.values
self.features = torch.tensor(features, dtype=self.dtype)
 self.feature_columns = feature_columns
if targets is None:
targets = -np.ones(self.features.shape[0])
 elif isinstance(targets,(pd.DataFrame, pd.Series)) :
targets = targets.values
self.targets = torch.tensor(targets, dtype=self.dtype)
 def __getitem__(self, i):
return self.features[i], self.targets[i]
def __len__(self):
return self.features.shape[0]
PoolIndex = df_all_1[df_all_1['PoolArea']!=0].index
df_all_1.loc[np.logical_not(df_all_1.index.isin(PoolIndex)) ,'PoolQC'] = 'NA'
df_all_1.loc[PoolIndex,'PoolQC'].isnull().sum()
df_all_1['PoolQC'] = df_all_1['PoolQC'].fillna(df_all_1.PoolQC[PoolIndex].mode() [0])
 df_all_1.PoolQC.value_counts()<feature_engineering>
def prepare_data(features, targets=None, OHE=True, scale=True, scaler=StandardScaler()):
zero_moa_mask =(features.cp_type == 'ctl_vehicle' ).values
features_enc = features[~zero_moa_mask].copy()
if scale:
features_enc.cp_time = features_enc.cp_time.astype(float)
 float_mask = features_enc.dtypes == 'float64'
if targets is not None:
scaler.fit(features_enc.loc[:, float_mask])
 features_enc.loc[:, float_mask] = scaler.transform(features_enc.loc[:, float_mask])
 if OHE:
features_enc.cp_type =(features_enc.cp_type == 'ctl_vehicle' ).astype(float)
 features_enc.cp_dose =(features_enc.cp_dose == 'D2' ).astype(float)
 features_enc = features_enc.drop(columns=['sig_id'])
 if targets is None:
return features_enc.values, zero_moa_mask
targets_enc = targets.copy() [~zero_moa_mask].drop(columns=['sig_id'])
 return features_enc.values, targets_enc.values
FireplaceIndex = df_all_1[df_all_1['Fireplaces']==0].index
df_all_1.loc[df_all_1.index.isin(FireplaceIndex),'FireplaceQu'] = 'NA'
df_all_1.loc[FireplaceIndex,'PoolQC'].isnull().sum() 
<count_unique_values>
train_features_enc, train_targets_scored_enc = prepare_data(train_features, train_targets_scored)
 test_features_enc, test_zero_moa_mask = prepare_data(test_features, OHE=True)
 train_features_enc
cat_length = 30
print('Imputing non-numeric missing variables...')
 df_all_1 = df_all_1.apply(lambda x: x.fillna(x.mode() [0])if(x.dtype == "object" and len(x.unique())<= cat_length)else x, axis=0 )<sort_values>
pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )<count_values>
num_workers = 0
for col in remainMiss.index:
if(df_all_1[col].dtypes != 'object' and len(np.unique(df_all_1[col])) < cat_length):
print(df_all_1[col].value_counts() )<feature_engineering>
class MoAModel(nn.Module):
def __init__(self, device, dtype, num_in_features, num_hidden_features, num_out_features, dropout_rate=0.5):
super().__init__()
self.device = device
self.dtype = dtype
self.net = nn.Sequential(
 nn.BatchNorm1d(num_in_features),
nn.Dropout(0.2),
nn.utils.weight_norm(nn.Linear(num_in_features, 2 * num_hidden_features)) ,
nn.ReLU() ,
nn.BatchNorm1d(2 * num_hidden_features),
nn.Dropout(dropout_rate),
nn.utils.weight_norm(nn.Linear(2 * num_hidden_features, num_hidden_features)) ,
nn.ReLU() ,
nn.BatchNorm1d(num_hidden_features),
nn.Dropout(dropout_rate),
nn.utils.weight_norm(nn.Linear(num_hidden_features, num_out_features)) ,
).to(self.device)
 def forward(self, x):
return self.net(x.to(self.device, self.dtype))
print('Imputing 3 missing numeric variables as categorical varaibles...')
 for col in ['BsmtHalfBath', 'BsmtFullBath','GarageCars']:
df_all_1[col] = df_all_1[col].fillna(df_all_1[col].mode() [0])
 pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )<feature_engineering>
def print_results(cur_results, mode, cur_iter, print_every):
if print_every == 'summary':
print(
 f'Summary: epoch {cur_results.epoch + 1:3}, '
f'mode {mode:6}, ',
end='' 
)
 if mode != 'test':
losses = cur_results.train_loss if mode == 'train' else cur_results.val_loss
print(
 f'loss {np.mean(losses):12.5f}, ' 
)
 else:
print()
elif cur_iter % print_every == 0:
print(
 f'Epoch {cur_results.epoch + 1:3}, '
f'mode {mode:6}, '
f'iter {cur_iter:5}, ',
end='' 
)
 if mode != 'test':
losses = cur_results.train_loss if mode == 'train' else cur_results.val_loss
print(
 f'loss {losses[-1]:12.5f}, ' 
)
 else:
print()
print("Imputing GarageYrBlt...")
 temp = df_all_1[df_all_1.GarageYrBlt.isnull() == True].index
df_all_1["GarageYrBlt"]= np.where(( df_all_1.GarageYrBlt.isnull() == True), df_all_1.YearBuilt,df_all_1.GarageYrBlt ).astype('int64')
 print('Chekcing if GarageYrBlt is corretly imputed...')
(df_all_1.loc[temp,"GarageYrBlt"] == df_all_1.loc[temp,"YearBuilt"] ).unique()<create_dataframe>
@functools.total_ordering
class ModelWithScore:
def __init__(self, score, model):
self.score = score
self.model = copy.deepcopy(model)
 def __eq__(self, other):
return self.score == other.score
def __lt__(self, other):
return self.score < other.score
def __repr__(self):
return f'Model of type "{type(self.model ).__name__}" with score {self.score}'
imputeCol = []
for col in df_all_1.columns:
if(df_all_1[col].dtypes != 'object' and len(df_all_1[col].unique())>= cat_length):
imputeCol.append(col)
 print("Columns used for KNN imputation are:
" + str(imputeCol))
imputer = KNNImputer(n_neighbors=5, weights='uniform', metric='nan_euclidean')
 X = df_all_1[imputeCol]
print('Imputing remaining numerical variables with KNN...')
 Xtrans = imputer.fit_transform(X)
 XT = pd.DataFrame(data=Xtrans)
 XT.columns = imputeCol
df_all_1[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']] \
= XT[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']]
print('There are now ' + str(df_all_1.isnull().any().sum())+ ' missing values in our data set.' )<sort_values>
def train_NN(model, criterion, optimizer, lr_scheduler,
max_epoch, print_every,
train_dataloader, val_dataloader, test_features_enc, test_zero_moa_mask):
results = Namespace(
 epochs=[],
train_loss=[],
val_loss=[],
test_preds=[],
best_models=[],
model=model,
criterion=criterion,
optimizer=optimizer,
lr_scheduler=lr_scheduler,
max_epoch=max_epoch,
init_lr=lr,
print_every=print_every,
train_dataloader=train_dataloader,
val_dataloader=val_dataloader,
test_features_enc=test_features_enc 
)
 for epoch in tqdm(range(max_epoch), desc='Epoch'):
results.epochs.append(epoch)
 model.train()
train_loss = 0
for cur_iter,(x, y)in enumerate(train_dataloader):
optimizer.zero_grad()
scores = model(x)
 loss = criterion(scores, y.to(device))
loss.backward()
optimizer.step()
train_loss += loss.item()
train_loss /= len(train_dataloader)
 results.train_loss.append(train_loss)
 model.eval()
val_loss = 0
for cur_iter,(x, y)in enumerate(val_dataloader):
with torch.no_grad() :
scores = model(x)
 loss = criterion(scores, y.to(device))
val_loss += loss.item()
val_loss /= len(val_dataloader)
 results.val_loss.append(val_loss)
 results.best_models.append(ModelWithScore(val_loss, model))
if len(results.best_models)> 3:
results.best_models.remove(max(results.best_models))
print(f'Epoch {epoch + 1:2}, train loss {train_loss:7.5f}, val loss {val_loss:7.5f}')
 test_preds = predict(model, test_features_enc=test_features_enc, zero_moa_mask=test_zero_moa_mask)
 results.test_preds.append(test_preds)
 return results
numVal =(( df_all_1.dtypes != 'object' ).index)[df_all_1.dtypes != 'object']
length = []
for col in numVal:
length.append(len(df_all_1[col].unique()))
numVal_df = pd.DataFrame({'Variables':numVal,'UniqueValuesLength':length})
 num_cat_Val = numVal_df[numVal_df['UniqueValuesLength'] < cat_length].sort_values(by='UniqueValuesLength', ascending=False)
 num_cat_Val.head()<filter>
def predict(model, test_features=None, test_features_enc=None, zero_moa_mask=None):
if test_features is not None:
test_features_enc, zero_moa_mask = prepare_data(test_features, OHE=True)
 test_dataset = MoADataset(dtype, test_features_enc)
 test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
 probs = []
model.eval()
for x, y in test_dataloader:
with torch.no_grad() :
scores = model(x)
 cur_probs = torch.sigmoid(scores ).cpu()
probs.append(cur_probs)
 probs = torch.cat(probs)
 answer = pd.read_csv(data_path + 'sample_submission.csv')
 answer.iloc[~zero_moa_mask, 1:] = probs
answer.iloc[zero_moa_mask, 1:] = 0
return answer
print('There are %i' %numVal_df[numVal_df['UniqueValuesLength'] < cat_length].shape[0] + ' numerical variables and %i' \
%(df_all_1.dtypes == 'object' ).sum() + ' non-numerical variables can potentially be treated as categorcial variables with some exceptions.' )<sort_values>
num_in_features = train_features_enc.shape[1]
num_out_features = train_targets_scored_enc.shape[1]
num_hidden_features = 512 * 2
cont_val = numVal_df[numVal_df['UniqueValuesLength'] >= cat_length].sort_values(by='UniqueValuesLength')
 cont_val.head()<sort_values>
print_every = 5
max_epoch = 45
lr = 1e-3
criterion = nn.BCEWithLogitsLoss()
yCor = cor['SalePrice'].sort_values(ascending = False ).reset_index().drop(index=0)
 yCor.columns = ["Variables", "Cor"]
yCor[(yCor["Cor"] > 0.5)|(yCor["Cor"] < -0.5)]<create_dataframe>
def create_model() :
model = MoAModel(device, dtype, num_in_features, num_hidden_features, num_out_features)
 optimizer = optim.Adam(model.parameters() , lr=lr, weight_decay=1e-5)
 lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=26, gamma=0.1)
 return model, optimizer, lr_scheduler
df_all_red = df_all_1.copy()<feature_engineering>
batch_size = 2 ** 7
n_folds = 7
df_all_red.YearRemodAdd = np.where(df_all_1.YearBuilt >df_all_1.YearRemodAdd, df_all_1.YearBuilt, df_all_1.YearRemodAdd)
 df_all_red.YrSold = np.where(df_all_1.YearBuilt >df_all_1.YrSold , df_all_1.YearBuilt, df_all_1.YrSold)
 df_all_red.YrSold = np.where(df_all_red.YearRemodAdd > df_all_red.YrSold, df_all_1.YearRemodAdd, df_all_red.YrSold)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt < df_all_red.YearBuilt, df_all_red.YearBuilt, df_all_red.GarageYrBlt)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt >2010, df_all_red.YearBuilt,df_all_red.GarageYrBlt)
 df_all_red['RestorePeriod'] = df_all_red.YearRemodAdd - df_all_red.YearBuilt
df_all_red['HouseAge'] = df_all_red.YrSold - df_all_red.YearBuilt
df_all_red["RemodAge"] = df_all_red.YrSold - df_all_red.YearRemodAdd
df_all_red['GarageAge'] = df_all_red.YrSold - df_all_red.GarageYrBlt
df_all_red["IsNewHouse"] = np.where(df_all_red.YearBuilt == df_all_red.YrSold, 1 ,0)
 df_all_red["Remodeled"] = np.where(df_all_red.RestorePeriod != 0, 1 ,0)
 df_all_red["GreatDepression"] = np.where(( df_all_1.YearBuilt >=1929)&(df_all_1.YearBuilt <= 1939), 1,0)
 df_all_red["Pre-GreatDepression"] = np.where(( df_all_1.YearBuilt <1929), 1,0)
 df_all_red["MortgageCrisis"] = np.where(( df_all_1.YearBuilt >=2007)&(df_all_1.YearBuilt <= 2010), 1,0)
 df_all_red["MortgageCrisisRestoration"] = np.where(( df_all_1.YearRemodAdd >=2007)&(df_all_1.YearRemodAdd <= 2010), 1,0)
<drop_column>
def train_NN_by_folds(n_folds, models_path):
Path(models_path ).mkdir(parents=True, exist_ok=False)
 kfold = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=517)
 models = []
for i,(train, val)in enumerate(tqdm(
 kfold.split(train_features_enc, train_targets_scored_enc),
total=kfold.n_splits 
)) :
print()
print(f'\tFold {i + 1}')
 x_tr = train_features_enc[train]
y_tr = train_targets_scored_enc[train]
x_val = train_features_enc[val]
y_val = train_targets_scored_enc[val]
train_dataset = MoADataset(dtype, x_tr, y_tr)
 val_dataset = MoADataset(dtype, x_val, y_val)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
 val_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
 model, optimizer, lr_scheduler = create_model()
result = train_NN(model, criterion, optimizer, lr_scheduler,
max_epoch, print_every,
train_dataloader, val_dataloader, test_features_enc, test_zero_moa_mask)
 np.save(f'{models_path}/best_models_kfold_{i + 1}', result.best_models)
 models.append(model )
df_all_red.drop(["YearBuilt","GarageYrBlt", "YearRemodAdd"], axis = 1, inplace = True )<feature_engineering>
models_path = '.. /input/nn-top3ensemble-with-cv7/nn-models/02/'
df_all_red["TotalFlrSF"] = df_all_red["1stFlrSF"] + df_all_red["2ndFlrSF"]
df_all_red["TotalBsmtFin"] = df_all_red['BsmtFinSF1']+df_all_red['BsmtFinSF2']
df_all_red["BsmtFinRatio"] =(df_all_red['TotalBsmtFin'] / df_all_red['TotalBsmtSF'] ).fillna(0)
 df_all_red["IsBasement"] = np.where(df_all_red['TotalBsmtSF'] < 1 , 0, 1)
 df_all_red["PorchArea"] = df_all_red['OpenPorchSF'] + df_all_red['EnclosedPorch'] + df_all_red['ScreenPorch'] + \
df_all_red["3SsnPorch"] + df_all_red['WoodDeckSF']
df_all_red["TotalHouseArea"] = df_all_red['TotalFlrSF'] + df_all_red['TotalBsmtSF']
df_all_red["TotalLivArea"] = df_all_red['GrLivArea'] + df_all_red['TotalBsmtSF']
df_all_red["2ndFloor"] = np.where(( df_all_red['2ndFlrSF'] < 1), 0, 1)
 df_all_red["GarageAreaPerCar"] =(df_all_red.GarageArea / df_all_red.GarageCars ).fillna(0)
 df_all_red["HouseLotRatio"] = df_all_red.TotalHouseArea /df_all_red.LotArea
df_all_red["LivLotRatio"] = df_all_red.GrLivArea / df_all_red.LotArea
df_all_red["GarageLotRatio"] = df_all_red.GarageArea / df_all_red.LotArea
df_all_red["MasVnrRatio"] = df_all_red.MasVnrArea / df_all_red.TotalHouseArea
df_all_red["DifArea"] =(df_all_red.LotArea - df_all_red["1stFlrSF"] - df_all_red.GarageArea - df_all_red.PorchArea)
 df_all_red["LowQualFinSFRatio"] = df_all_red.LowQualFinSF/df_all_red.TotalHouseArea<count_values>
all_preds = []
for i in tqdm(range(n_folds)) :
results = np.load(f'{models_path}/best_models_kfold_{i + 1}.npy', allow_pickle=True)
 for best_model in results:
preds = predict(best_model.model, test_features_enc=test_features_enc, zero_moa_mask=test_zero_moa_mask)
 all_preds.append(preds.iloc[:, 1:].values )
for col in ['TotRmsAbvGrd', 'OverallQual', 'OverallCond','BedroomAbvGr','GarageCars','KitchenAbvGr','FullBath','MissingCount']:
print(col + ' has ' + str(len(df_all_red[col].value_counts())) + ' unique numerical values.' )<feature_engineering>
answer = preds.copy()
answer.iloc[:, 1:] = np.array(all_preds ).mean(axis=0 )
for col in ['OverallQual','OverallCond']:
df_all_red[col].replace(np.linspace(1,10,10 ).astype(int), list(np.linspace(10,100,10)) , inplace=True)
 df_all_red['OverallScore'] = df_all_red[['OverallQual', 'OverallCond']].sum(axis = 1)/2
df_all_red["OverallGrade"] = df_all_red["OverallQual"] * df_all_red["OverallCond"]<categorify>
answer.to_csv('submission.csv', index=False )