Kaggle competition: Restaurant Revenue Prediction
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<train_model>
df = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')
 df.shape
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
test_df = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')
 test_df.shape
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
df.isnull().sum().sort_index() /len(df )
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions<feature_engineering>
( df['City'].nunique() , test_df['City'].nunique() )
str(timedelta(seconds=time_diff))<compute_test_metric>
test_df.loc[test_df['Type']=='MB', 'Type'] = 'DT'
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<merge>
df.drop('City', axis=1, inplace=True)
 test_df.drop('City', axis=1, inplace=True )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )<prepare_output>
df.drop('Id',axis=1,inplace=True)
 df['Open Date'] = pd.to_datetime(df['Open Date'])
 test_df['Open Date'] = pd.to_datetime(test_df['Open Date'])
 launch_date = datetime.datetime(2015, 3, 23)
 df['Days Open'] =(launch_date - df['Open Date'] ).dt.days / 1000
test_df['Days Open'] =(launch_date - test_df['Open Date'] ).dt.days / 1000
df.drop('Open Date', axis=1, inplace=True)
 test_df.drop('Open Date', axis=1, inplace=True )
sub9 = sub.copy()<set_options>
warnings.filterwarnings('ignore' )<load_from_csv>
imp_train = IterativeImputer(max_iter=30, missing_values=0, sample_posterior=True, min_value=1, random_state=37)
 imp_test = IterativeImputer(max_iter=30, missing_values=0, sample_posterior=True, min_value=1, random_state=23)
 p_data = ['P'+str(i)for i in range(1,38)]
df[p_data] = np.round(imp_train.fit_transform(df[p_data]))
test_df[p_data] = np.round(imp_test.fit_transform(test_df[p_data]))
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
columnsToEncode = df.select_dtypes(include=[object] ).columns
df = pd.get_dummies(df, columns=columnsToEncode, drop_first=False)
 test_df = pd.get_dummies(test_df, columns=columnsToEncode, drop_first=False )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<import_modules>
df['revenue'] = np.log1p(df['revenue'])
 X, y = df.drop('revenue', axis=1), df['revenue']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=118 )
from sklearn.preprocessing import QuantileTransformer<normalization>
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
params_ridge = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
'solver' : ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']
}
ridge_model = Ridge()
ridge_regressor = GridSearchCV(ridge_model, params_ridge, scoring='neg_root_mean_squared_error', cv=5, n_jobs=-1)
 ridge_regressor.fit(X_train, y_train)
 print(f'Optimal alpha: {ridge_regressor.best_params_["alpha"]:.2f}')
 print(f'Optimal fit_intercept: {ridge_regressor.best_params_["fit_intercept"]}')
 print(f'Optimal normalize: {ridge_regressor.best_params_["normalize"]}')
 print(f'Optimal solver: {ridge_regressor.best_params_["solver"]}')
 print(f'Best score: {ridge_regressor.best_score_}' )
def seed_everything(seed=321):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=321 )<concatenate>
ridge_model = Ridge(alpha=ridge_regressor.best_params_["alpha"], fit_intercept=ridge_regressor.best_params_["fit_intercept"],
normalize=ridge_regressor.best_params_["normalize"], solver=ridge_regressor.best_params_["solver"])
 ridge_model.fit(X_train, y_train)
 y_train_pred = ridge_model.predict(X_train)
 y_pred = ridge_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
n_comp = 29
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data_PCA = PCA(n_components=n_comp, random_state=42 ).fit(data[GENES][:train_features.shape[0]])
 data2 =(data_PCA.transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data_PCA =(PCA(n_components=n_comp, random_state=42 ).fit(data[CELLS][:train_features.shape[0]]))
data2 =(data_PCA.transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<prepare_x_and_y>
params_lasso = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
}
lasso_model = Lasso()
lasso_regressor = GridSearchCV(lasso_model, params_lasso, scoring='neg_root_mean_squared_error', cv=5, n_jobs=-1)
 lasso_regressor.fit(X_train, y_train)
 print(f'Optimal alpha: {lasso_regressor.best_params_["alpha"]:.2f}')
 print(f'Optimal fit_intercept: {lasso_regressor.best_params_["fit_intercept"]}')
 print(f'Optimal normalize: {lasso_regressor.best_params_["normalize"]}')
 print(f'Best score: {lasso_regressor.best_score_}' )
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 data_transformed_var = var_thresh.fit(data.iloc[:, 4:][:train_features.shape[0]])
 data_transformed = data_transformed_var.transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features 
<feature_engineering>
lasso_model = Lasso(alpha=lasso_regressor.best_params_["alpha"], fit_intercept=lasso_regressor.best_params_["fit_intercept"],
normalize=lasso_regressor.best_params_["normalize"])
 lasso_model.fit(X_train, y_train)
 y_train_pred = lasso_model.predict(X_train)
 y_pred = lasso_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )<merge>
el_model = ElasticNetCV(l1_ratio=[.1,.5,.7,.9,.95,.99, 1], eps=5e-2, cv=10, n_jobs=-1)
 el_model.fit(X_train, y_train)
 print(f'Optimal alpha: {el_model.alpha_:.6f}')
 print(f'Optimal l1_ratio: {el_model.l1_ratio_:.3f}')
 print(f'Number of iterations {el_model.n_iter_}' )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
y_train_pred = el_model.predict(X_train)
 y_pred = el_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist() 
<split>
params_knn = {
'n_neighbors' : [3, 5, 7, 9, 11],
}
knn_model = KNeighborsRegressor()
knn_regressor = GridSearchCV(knn_model, params_knn, scoring='neg_root_mean_squared_error', cv=10, n_jobs=-1)
 knn_regressor.fit(X_train, y_train)
 print(f'Optimal neighbors: {knn_regressor.best_params_["n_neighbors"]}')
 print(f'Best score: {knn_regressor.best_score_}' )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<categorify>
knn_model = KNeighborsRegressor(n_neighbors=knn_regressor.best_params_["n_neighbors"])
 knn_model.fit(X_train, y_train)
 y_train_pred = knn_model.predict(X_train)
 y_pred = knn_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
params_rf = {
'max_depth': [10, 30, 35, 50, 65, 75, 100],
'max_features': [.3,.4,.5,.6],
'min_samples_leaf': [3, 4, 5],
'min_samples_split': [8, 10, 12],
'n_estimators': [30, 50, 100, 200]
}
rf = RandomForestRegressor()
rf_regressor = GridSearchCV(rf, params_rf, scoring='neg_root_mean_squared_error', cv = 10, n_jobs = -1)
 rf_regressor.fit(X_train, y_train)
 print(f'Optimal depth: {rf_regressor.best_params_["max_depth"]}')
 print(f'Optimal max_features: {rf_regressor.best_params_["max_features"]}')
 print(f'Optimal min_sample_leaf: {rf_regressor.best_params_["min_samples_leaf"]}')
 print(f'Optimal min_samples_split: {rf_regressor.best_params_["min_samples_split"]}')
 print(f'Optimal n_estimators: {rf_regressor.best_params_["n_estimators"]}')
 print(f'Best score: {rf_regressor.best_score_}' )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
rf_model = RandomForestRegressor(max_depth=rf_regressor.best_params_["max_depth"],
max_features=rf_regressor.best_params_["max_features"],
min_samples_leaf=rf_regressor.best_params_["min_samples_leaf"],
min_samples_split=rf_regressor.best_params_["min_samples_split"],
n_estimators=rf_regressor.best_params_["n_estimators"],
n_jobs=-1, oob_score=True)
 rf_model.fit(X_train, y_train)
 y_train_pred = rf_model.predict(X_train)
 y_pred = rf_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
params_lgbm = {
'learning_rate': [.01,.1,.5,.7,.9,.95,.99, 1],
'boosting': ['gbdt'],
'metric': ['l1'],
'feature_fraction': [.3,.4,.5, 1],
'num_leaves': [20],
'min_data': [10],
'max_depth': [10],
'n_estimators': [10, 30, 50, 100]
}
lgb = lgbm.LGBMRegressor()
lgb_regressor = GridSearchCV(lgb, params_lgbm, scoring='neg_root_mean_squared_error', cv = 10, n_jobs = -1)
 lgb_regressor.fit(X_train, y_train)
 print(f'Optimal lr: {lgb_regressor.best_params_["learning_rate"]}')
 print(f'Optimal feature_fraction: {lgb_regressor.best_params_["feature_fraction"]}')
 print(f'Optimal n_estimators: {lgb_regressor.best_params_["n_estimators"]}')
 print(f'Best score: {lgb_regressor.best_score_}' )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.activation1 = torch.nn.PReLU(num_parameters = hidden_size, init = 1.0)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.3)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.activation2 = torch.nn.PReLU(num_parameters = hidden_size, init = 1.0)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def recalibrate_layer(self, layer):
if(torch.isnan(layer.weight_v ).sum() > 0):
print('recalibrate layer.weight_v')
 layer.weight_v = torch.nn.Parameter(torch.where(torch.isnan(layer.weight_v), torch.zeros_like(layer.weight_v), layer.weight_v))
layer.weight_v = torch.nn.Parameter(layer.weight_v + 1e-7)
 if(torch.isnan(layer.weight ).sum() > 0):
print('recalibrate layer.weight')
 layer.weight = torch.where(torch.isnan(layer.weight), torch.zeros_like(layer.weight), layer.weight)
 layer.weight += 1e-7
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 self.recalibrate_layer(self.dense1)
 x = self.activation1(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 self.recalibrate_layer(self.dense2)
 x = self.activation2(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 self.recalibrate_layer(self.dense3)
 x = self.dense3(x)
 return x<feature_engineering>
lgb_model = lgbm.LGBMRegressor(learning_rate=lgb_regressor.best_params_["learning_rate"], boosting='gbdt',
metric='l1', feature_fraction=lgb_regressor.best_params_["feature_fraction"],
num_leaves=20, min_data=10, max_depth=10,
n_estimators=lgb_regressor.best_params_["n_estimators"], n_jobs=-1)
 lgb_model.fit(X_train, y_train)
 y_train_pred = lgb_model.predict(X_train)
 y_pred = lgb_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
params_xgb = {
'learning_rate': [.1,.5,.7,.9,.95,.99, 1],
'colsample_bytree': [.3,.4,.5,.6],
'max_depth': [4],
'alpha': [3],
'subsample': [.5],
'n_estimators': [30, 70, 100, 200]
}
xgb_model = XGBRegressor()
xgb_regressor = GridSearchCV(xgb_model, params_xgb, scoring='neg_root_mean_squared_error', cv = 10, n_jobs = -1)
 xgb_regressor.fit(X_train, y_train)
 print(f'Optimal lr: {xgb_regressor.best_params_["learning_rate"]}')
 print(f'Optimal colsample_bytree: {xgb_regressor.best_params_["colsample_bytree"]}')
 print(f'Optimal n_estimators: {xgb_regressor.best_params_["n_estimators"]}')
 print(f'Best score: {xgb_regressor.best_score_}' )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<prepare_x_and_y>
xgb_model = XGBRegressor(learning_rate=xgb_regressor.best_params_["learning_rate"],
colsample_bytree=xgb_regressor.best_params_["colsample_bytree"],
max_depth=4, alpha=3, subsample=.5,
n_estimators=xgb_regressor.best_params_["n_estimators"], n_jobs=-1)
 xgb_model.fit(X_train, y_train)
 y_train_pred = xgb_model.predict(X_train)
 y_pred = xgb_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
def run_pred(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/kubipytorchrg/FOLD{fold}_.pth",map_location=torch.device('cpu')))
 model.to(DEVICE)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.0001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 del model,testdataset,testloader,y_train,y_valid,validloader,trn_idx,val_idx,train,train_dataset,valid_dataset,valid_loss,best_loss,test_,loss_fn,loss_tr,trainloader,x_train,x_valid,
gc.collect()
return oof, predictions<categorify>
xgb_model.fit(X_train, y_train, early_stopping_rounds=4,
eval_set=[(X_test, y_test)], verbose=False)
 y_train_pred = xgb_model.predict(X_train)
 y_pred = xgb_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_pred(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
rf_model_en = RandomForestRegressor(max_depth=200, max_features=0.4, min_samples_leaf=3,
min_samples_split=6, n_estimators=30, n_jobs=-1, oob_score=True)
 rf_model_en.fit(X_train, y_train)
 y_train_pred = rf_model_en.predict(X_train)
 y_pred = rf_model_en.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 512
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 25
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<feature_engineering>
def get_stacking() :
base_models = list()
base_models.append(( 'ridge', ridge_model))
base_models.append(( 'lasso', lasso_model))
base_models.append(( 'rf', rf_model_en))
learner = LinearRegression()
model = StackingRegressor(estimators=base_models, final_estimator=learner, cv=10)
 return model
def get_models() :
models = dict()
models['ridge'] = ridge_model
models['lasso'] = lasso_model
models['rf_en'] = rf_model_en
models['stacking'] = get_stacking()
return models
def evaluate_model(model, X, y):
cv = RepeatedKFold(n_splits=10, n_repeats=5, random_state=19)
 scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1, error_score='raise')
 return scores
models = get_models()
results, names = list() , list()
for name, model in models.items() :
scores = evaluate_model(model, X_train, y_train)
 results.append(scores)
 names.append(name)
 print(f'{name} {mean(scores):.3f} {std(scores):.3f}')
 pyplot.boxplot(results, labels=names, showmeans=True)
 pyplot.show()
SEED = [456]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
base_models = list()
base_models.append(( 'ridge', ridge_model))
base_models.append(( 'lasso', lasso_model))
base_models.append(( 'rf', rf_model_en))
learner = LinearRegression()
stack1 = StackingRegressor(estimators=base_models, final_estimator=learner, cv=10)
 stack1.fit(X, y )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<merge>
base_model = list()
base_model.append(( 'rf1', rf_model))
base_model.append(( 'rf2', rf_model_en))
base_model.append(( 'rf3', RandomForestRegressor(max_depth=8, max_features=0.1, min_samples_leaf=3,
min_samples_split=2, n_estimators=250, n_jobs=-1, oob_score=False)))
 learner = LinearRegression()
stack2 = StackingRegressor(estimators=base_model, final_estimator=learner, cv=10)
 stack2.fit(X, y )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )<prepare_output>
submission = pd.DataFrame(columns=['Id','Prediction'])
 submission['Id'] = test_df['Id']
ridge_pred = ridge_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(ridge_pred)
 submission.to_csv('submission_ridge.csv',index=False)
 lasso_pred = lasso_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(lasso_pred)
 submission.to_csv('submission_lasso.csv',index=False)
 elastic_pred = el_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(elastic_pred)
 submission.to_csv('submission_elastic.csv',index=False)
 knn_pred = knn_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(knn_pred)
 submission.to_csv('submission_knn.csv',index=False)
 rf_pred = rf_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(rf_pred)
 submission.to_csv('submission_rf.csv',index=False)
 lgb_pred = lgb_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(lgb_pred)
 submission.to_csv('submission_lgb.csv',index=False)
 xgb_pred = xgb_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(xgb_pred)
 submission.to_csv('submission_xgb.csv',index=False)
 stack_pred1 = stack1.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(stack_pred1)
 submission.to_csv('submission_stack1.csv',index=False)
 stack_pred2 = stack2.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(stack_pred2)
 submission.to_csv('submission_stack2.csv',index=False )