Kaggle competition: Mechanisms of Action (MoA) Prediction
xgboost_test = XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 xgboost_test.fit(X_sub_train, Y_sub_train)
 Y_fit_xgb = xgboost_test.predict(X_sub_test)
 rmse_xgb = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_xgb))
rmse_xgb<compute_train_metric>
warnings.filterwarnings('ignore' )
estimators = [('ridge', ridge_test), 
('lasso', lasso_test), 
('lgbm', lgbm_test), 
('elastic', elastic_test), 
('xgb', xgboost_test)]
stack = StackingRegressor(estimators = estimators)
 stack.fit(X_sub_train,Y_sub_train)
 Y_fit_stack = stack.predict(X_sub_test)
 rmse_stack = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_stack))
rmse_stack<save_to_csv>
from sklearn.preprocessing import QuantileTransformer
stack_final = StackingRegressor(estimators = estimators)
 stack_final.fit(X_train, Y_train)
 Y_fit_final = stack_final.predict(X_test)
 pred = np.expm1(Y_fit_final)
 submission = pd.DataFrame({'Id':test_all['Id'], 'SalePrice':pred})
 submission.to_csv("submission.csv", index=False )<set_options>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
np.set_printoptions(suppress=True)
 pio.templates.default = "plotly_white" 
<load_from_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train_data=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 pd.set_option('display.max_columns', 500)
 print(train_data.shape)
 train_data.describe()<prepare_x_and_y>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
X_train = train_data.loc[:,'MSSubClass':'SaleCondition']
y_train = train_data.SalePrice
X_test = test_data.loc[:,'MSSubClass':'SaleCondition']<create_dataframe>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
def missing_ratio(all_data):
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 return missing_data<concatenate>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
train_na=missing_ratio(X_train)
 test_na=missing_ratio(X_test)
 compare_na = pd.concat([train_na, test_na], axis=1)
 compare_na.columns=['Train Missing Ratio','Test Missing Ratio']
compare_na.fillna(0 )<filter>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
drop_col=list(compare_na[(compare_na['Train Missing Ratio'] > 99)|(compare_na['Test Missing Ratio'] >99)].index)
 drop_col<categorify>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
def manu_miss(all_data):
for NA_to_0 in('MasVnrArea' ,'GarageYrBlt', 'GarageArea', 'GarageCars','BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[NA_to_0] = all_data[NA_to_0].fillna(0)
 for NA_to_None in('MiscFeature','Alley','Fence','FireplaceQu','GarageType', 'GarageFinish','GarageQual', 'GarageCond','BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2','MasVnrType'):
all_data[NA_to_None] = all_data[NA_to_None].fillna('None')
 all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['LotFrontage']=all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x:x.fillna(x.median()))
all_data['MSZoning']=all_data.groupby('Neighborhood')['MSZoning'].transform(lambda x:x.fillna(x.mode() [0]))
all_data['Electrical']=all_data.groupby('YearRemodAdd')['Electrical'].transform(lambda x:x.fillna(x.mode() [0]))
all_data['KitchenQual']=all_data.groupby('ExterQual')['KitchenQual'].transform(lambda x:x.fillna(x.mode() [0]))
all_data['SaleType']=all_data.groupby('SaleCondition')['SaleType'].transform(lambda x:x.fillna(x.mode() [0]))
return all_data<compute_test_metric>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
X_train=manu_miss(X_train)
 X_test=manu_miss(X_test )<concatenate>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
X_train = pd.concat([imputed_X_train_num, imputed_X_train_cat], axis=1)
 X_test = pd.concat([imputed_X_test_num, imputed_X_test_cat], axis=1)
 X_train.shape<categorify>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
def ordered_feature_transform(all_data):
QualMap={'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'None':0}
Qualmap_features=('ExterQual','ExterCond','BsmtQual','BsmtCond','HeatingQC','KitchenQual','FireplaceQu','GarageQual','GarageCond')
 for map_feature in Qualmap_features:
all_data[map_feature]=all_data[map_feature].map(QualMap)
 BsmtExposureMap={'Gd':4,'Av':3,'Mn':2,'No':1,'None':0}
all_data['BsmtExposure']=all_data['BsmtExposure'].map(BsmtExposureMap)
 BsmtFinTypeMap={'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'None':0}
BsmtFinTypemap_features=('BsmtFinType1','BsmtFinType2')
 for map_feature in BsmtFinTypemap_features:
all_data[map_feature]=all_data[map_feature].map(BsmtFinTypeMap)
 UtilitiesMap={'AllPub':4,'NoSewr':3,'NoSeWa':2,'ELO':1,'None':0}
all_data['Utilities']=all_data['Utilities'].map(UtilitiesMap)
 GarageFinishMap={'Fin':3,'RFn':2,'Unf':1,'None':0}
all_data['GarageFinish']=all_data['GarageFinish'].map(GarageFinishMap)
 CentralAirMap={'Y':1,'N':0}
all_data['CentralAir']=all_data['CentralAir'].map(CentralAirMap)
 MSSubClassMap={'180':1, '30':2, '45':2,'190':3, '50':3, '90':3,'85':4, '40':4, '160':4,'70':5, '20':5, '75':5, '80':5, '150':5,'120': 6, '60':6}
all_data['MSSubClass'] = all_data['MSSubClass'].astype(int ).apply(str)
 all_data['MSSubClass']=all_data['MSSubClass'].map(MSSubClassMap)
 MSZoningMap={'C(all)':1, 'RH':2, 'RM':2, 'RL':3, 'FV':4}
all_data['MSZoning']=all_data['MSZoning'].map(MSZoningMap)
 NeighborhoodMap={'MeadowV':1,'IDOTRR':2, 'BrDale':2,'OldTown':3, 'Edwards':3, 'BrkSide':3,'Sawyer':4, 'Blueste':4, 'SWISU':4, 'NAmes':4,
'NPkVill':5, 'Mitchel':5,'SawyerW':6, 'Gilbert':6, 'NWAmes':6,'Blmngtn':7, 'CollgCr':7, 'ClearCr':7, 'Crawfor':7,
'Veenker':8, 'Somerst':8, 'Timber':8,'StoneBr':9,'NoRidge':10, 'NridgHt':10}
all_data['Neighborhood']=all_data['Neighborhood'].map(NeighborhoodMap)
 HouseStyleMap={'1.5Unf':1,'1.5Fin':2, '2.5Unf':2, 'SFoyer':2, '1Story':3, 'SLvl':3,'2Story':4, '2.5Fin':4}
all_data['HouseStyle']=all_data['HouseStyle'].map(HouseStyleMap)
 MasVnrTypeMap={'BrkCmn':1, 'None':1, 'BrkFace':2, 'Stone':3}
all_data['MasVnrType']=all_data['MasVnrType'].map(MasVnrTypeMap)
 FoundationMap={'Slab':1, 'BrkTil':2, 'CBlock':2, 'Stone':2,'Wood':3, 'PConc':4}
all_data['Foundation']=all_data['Foundation'].map(FoundationMap)
 GarageTypeMap={'CarPort':1, 'None':1,'Detchd':2,'2Types':3, 'Basment':3,'Attchd':4, 'BuiltIn':5}
all_data['GarageType']=all_data['GarageType'].map(GarageTypeMap)
 SaleConditionMap={'AdjLand':1, 'Abnorml':2, 'Alloca':2, 'Family':2, 'Normal':3, 'Partial':4}
all_data['SaleCondition']=all_data['SaleCondition'].map(SaleConditionMap)
 return all_data<feature_engineering>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
X_train=ordered_feature_transform(X_train)
 X_test=ordered_feature_transform(X_test )<feature_engineering>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def add_new_feature(all_data):
all_data['BsmtFin'] =all_data['BsmtFinType1']*all_data['BsmtFinSF1']+all_data['BsmtFinType2']*all_data['BsmtFinSF2']
all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_data['LargeHouseLevel'] = all_data['GrLivArea'].apply(lambda x: 1 if x > 2750 else 0)
 all_data['OldHouse'] = all_data['YearBuilt'].apply(lambda x: 1 if x < 1910 else 0)
 all_data['BsmtUnfSF_cat'] = all_data['BsmtUnfSF'].apply(lambda x: 1 if x <=500 else 2 if x <= 1000 else 3 if x <= 1500 else 4 if x <= 2000 else 5)
 all_data['Qual']=all_data['BsmtQual']+all_data['HeatingQC']+all_data['KitchenQual']+all_data['FireplaceQu']+all_data['GarageQual']
all_data["AllSF"] = all_data["GrLivArea"] + all_data["TotalBsmtSF"]
all_data['Total_porch'] = all_data['OpenPorchSF']+ all_data['3SsnPorch'] + all_data['EnclosedPorch'] + all_data['ScreenPorch'] + all_data['WoodDeckSF']
all_data['YrBltAndRemod'] = all_data['YearBuilt'] + all_data['YearRemodAdd']
all_data['Total_Bathrooms'] = all_data['FullBath'] +(0.5 * all_data['HalfBath'])+ all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath'])
 all_data['Total_sqr_footage'] = all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
return all_data<feature_engineering>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
X_train=add_new_feature(X_train)
 X_test=add_new_feature(X_test )<drop_column>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def drop_rest(all_data):
all_data=all_data.drop(['BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','MiscFeature','1stFlrSF','2ndFlrSF','LowQualFinSF'], axis=1)
 return all_data<drop_column>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
X_train=drop_rest(X_train)
 X_test=drop_rest(X_test )<drop_column>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def manu_dum(all_data):
for new_col in('Artery','Feedr','Norm','RRNn','RRAn','PosN','PosA','RRNe','RRAe'):
all_data[new_col]=0.0
for col in('Condition1','Condition2'):
all_data.loc[all_data[col] == new_col, new_col] = 1.0
all_data=all_data.drop(['Condition1','Condition2'], axis=1)
 for new_col in('AsbShng','AsphShn','BrkComm','BrkFace','CBlock','CemntBd','HdBoard','ImStucc','MetalSd','Other','Plywood','Stone','Stucco','VinylSd','Wd Shng','WdShing'):
all_data[new_col]=0.0
for col in('Exterior1st','Exterior2nd'):
all_data.loc[all_data[col] == new_col, new_col] = 1.0
all_data=all_data.drop(['Exterior1st','Exterior2nd'], axis=1)
 return all_data<concatenate>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
X_train=manu_dum(X_train)
 X_test=manu_dum(X_test )<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
ObjectData=X_train.dtypes[X_train.dtypes == "object"].index
print(X_train[ObjectData].columns)
 cat_features=list(X_train[ObjectData].columns )<define_variables>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
cat_features=['Street', 'Alley', 'LotShape', 'LandContour', 'LotConfig', 'LandSlope',
'RoofStyle', 'RoofMatl', 'Heating', 'Electrical',
'Functional', 'PavedDrive', 'Fence', 'SaleType']<categorify>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
for cat_feature in cat_features:
encoder = LabelEncoder()
encoder.fit(list(X_train[cat_feature].values))
X_train[cat_feature] = encoder.transform(list(X_train[cat_feature].values))
X_test[cat_feature] = encoder.transform(list(X_test[cat_feature].values))<categorify>
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
X_train = pd.get_dummies(X_train)
 X_test = pd.get_dummies(X_test)
 X_train, X_test = X_train.align(X_test, join='left', axis=1, fill_value=0)
 print(X_train.shape,X_test.shape )<feature_engineering>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
numeric_feats = X_train.dtypes[X_train.dtypes != "object"].index
skewed_feats = X_train[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 print(skewness.head(30))
X_train[skewness.index] = np.log1p(X_train[skewness.index])
 X_test[skewness.index] = np.log1p(X_test[skewness.index])
 y_train= np.log1p(y_train )<train_model>
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets_scored.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics )
<predict_on_test><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )