Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAM
<SOS> metric: logloss Kaggle data source: google-cloud-ncaa-march-madness-2020-division-1-mens-tournament<save_to_csv>
lgb_num_leaves_max = 255
lgb_in_leaf = 50
lgb_lr = 0.0001
lgb_bagging = 7
xgb_max_depth = 20
xgb_min_child_weight = 75
xgb_lr = 0.0005
xgb_num_boost_round_max = 4000
w_lgb = 0.6
w_xgb = 0.3
w_logreg = 1 - w_lgb - w_xgb
w_logreg
output=pd.DataFrame()
output['ForecastId']=test['ForecastId'].astype(int)
 output['ConfirmedCases']=test['ConfirmedCases'].astype(int)
 output['Fatalities']=test['Fatalities'].astype(int)
 output.to_csv('submission.csv',index=False )<load_from_csv>
warnings.filterwarnings("ignore" )
train_dataset = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_dataset = pd.read_csv('/kaggle/input/titanic/test.csv')
 X_train = train_dataset.iloc[:, [False,False,True,False,True,True,True,True,False,True,False,False]].values
y_train = train_dataset.iloc[:, 1].values
X_test = test_dataset.iloc[:, [False,True,False,True,True,True,True,False,True,False,False]].values<categorify>
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 tourney_result = tourney_result[tourney_result['Season'] < 2015]
tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')
 tourney_seed = tourney_seed[tourney_seed['Season'] < 2015]
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1 )
ct = ColumnTransformer(transformers = [('encoder',OneHotEncoder() ,[1])],remainder = 'passthrough')
<normalization>
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
X_train = np.array(ct.fit_transform(X_train))
X_test = np.array(ct.fit_transform(X_test))<train_model>
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')
 season_result = season_result[season_result['Season'] < 2015]
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
imputer = SimpleImputer(missing_values = np.nan, strategy = 'mean')
 imputer.fit(X_train)
 X_train = imputer.transform(X_train)
 imputer.fit(X_test)
 X_test = imputer.transform(X_test )<normalization>
tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1 )
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test )<feature_engineering>
tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)
 tourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True )
for i in range(len(X_train)) :
if X_train[i][0]>X_train[i][1]:
X_train[i][0] = 1
X_train[i][1] = 0
else:
X_train[i][0] = 0
X_train[i][1] = 1
print(X_train )<feature_engineering>
tourney_lose_result = tourney_win_result.copy()
tourney_lose_result['Seed1'] = tourney_win_result['Seed2']
tourney_lose_result['Seed2'] = tourney_win_result['Seed1']
tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']
tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']
for i in range(len(X_test)) :
if X_test[i][0]>X_test[i][1]:
X_test[i][0] = 1
X_test[i][1] = 0
else:
X_test[i][0] = 0
X_test[i][1] = 1
print(X_test )<train_model>
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']
classifier = KNeighborsClassifier(n_neighbors=5, metric='minkowski',p=2)
 classifier.fit(X_train, y_train )<predict_on_test>
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
train_df = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True)
 train_df
y_test = classifier.predict(X_test)
 print(y_test )<save_to_csv>
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 sub = test_df.copy()
output = pd.DataFrame({'PassengerId': test_dataset.PassengerId, 'Survived': y_test})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
 class Model1Xgb:
def __init__(self):
self.model = None
def fit(self, tr_x, tr_y, va_x, va_y):
params = {
'booster': 'gbtree',
'objective': 'binary:logistic',
'eta': 0.1,
'gamma': 0.1,
'reg_alpha':0.01 ,
'reg_lambda': 0,
'min_child_weight': 4,
'max_depth': 9,
'subsample': 0.8,
'colsample_bytree': 0.75,
'scale_pos_weight': 1,
'random_state': 1
}
num_round = 30
dtrain = xgb.DMatrix(tr_x, label=tr_y)
 dvalid = xgb.DMatrix(va_x, label=va_y)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
self.model = xgb.train(params, dtrain, num_round, evals=watchlist)
 def predict(self, x):
data = xgb.DMatrix(x)
 pred = self.model.predict(data)
 return pred
class Model1NN:
def __init__(self, params):
self.params = params
self.model = None
self.scaler = None
def fit(self, tr_x, tr_y, va_x, va_y):
input_dropout = self.params['input_dropout']
hidden_layers = int(self.params['hidden_layers'])
 hidden_units = int(self.params['hidden_units'])
 hidden_activation = self.params['hidden_activation']
hidden_dropout = self.params['hidden_dropout']
batch_norm = self.params['batch_norm']
optimizer_type = self.params['optimizer']['type']
optimizer_lr = self.params['optimizer']['lr']
batch_size = int(self.params['batch_size'])
 self.scaler = StandardScaler()
self.scaler.fit(tr_x)
 batch_size = 32
epochs = 20
tr_x = self.scaler.transform(tr_x)
 va_x = self.scaler.transform(va_x)
 model = Sequential()
model.add(Dropout(input_dropout, input_shape=(tr_x.shape[1],)))
 for i in range(hidden_layers):
model.add(Dense(hidden_units))
if batch_norm == 'before_act':
model.add(BatchNormalization())
 if hidden_activation == 'prelu':
model.add(PReLU())
 elif hidden_activation == 'relu':
model.add(ReLU())
 else:
raise NotImplementedError
model.add(Dropout(hidden_dropout))
model.add(Dense(1, activation='sigmoid'))
if optimizer_type == 'sgd':
optimizer = SGD(lr=optimizer_lr, decay=1e-6, momentum=0.9, nesterov=True)
 elif optimizer_type == 'adam':
optimizer = Adam(lr=optimizer_lr, beta_1=0.9, beta_2=0.999, decay=0.)
else:
raise NotImplementedError
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 history = model.fit(tr_x, tr_y, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(va_x, va_y))
self.model = model
def predict(self, x):
x = self.scaler.transform(x)
 pred = self.model.predict_proba(x ).reshape(-1)
 return pred
class Model2Linear:
def __init__(self):
self.model = None
self.scaler = None
def fit(self, tr_x, tr_y, va_x, va_y):
self.scaler = StandardScaler()
self.scaler.fit(tr_x)
 tr_x = self.scaler.transform(tr_x)
 self.model = LogisticRegression(solver='lbfgs', C=100, random_state=1)
 self.model.fit(tr_x, tr_y)
 def predict(self, x):
x = self.scaler.transform(x)
 pred = self.model.predict_proba(x)[:, 1]
return pred
def predict_cv(model, train_x, train_y, test_x):
preds = []
preds_test = []
va_idxes = []
kf = KFold(n_splits=4, shuffle=True, random_state=1)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model.fit(tr_x, tr_y, va_x, va_y)
 pred = model.predict(va_x)
 preds.append(pred)
 pred_test = model.predict(test_x)
 preds_test.append(pred_test)
 va_idxes.append(va_idx)
 va_idxes = np.concatenate(va_idxes)
 preds = np.concatenate(preds, axis=0)
 order = np.argsort(va_idxes)
 pred_train = preds[order]
preds_test = np.mean(preds_test, axis=0)
 return pred_train, preds_test
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 train_y = train_df['Survived']
train_df = train_df.drop(["Ticket", "Cabin"], axis=1)
 test_df = test_df.drop(["Ticket", "Cabin"], axis=1)
 combine = [train_df, test_df]
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df_list = pd.crosstab(train_df['Title'], train_df['Sex'])
 for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
combine = [train_df, test_df]
for dataset in combine:
dataset["Title"] = dataset["Title"].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5})
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
 train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True)
 for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df = train_df.drop(['AgeBand'], axis=1)
 combine = [train_df, test_df]
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 for dataset in combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 combine = [train_df, test_df]
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
freq_port = train_df.Embarked.dropna().mode() [0]
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True)
 combine = [train_df, test_df]
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand','Survived'], axis=1)
 test_df = test_df.drop(['PassengerId'], axis=1)
 train_df['Age'] = train_df['Age'].fillna(train_df['Age'].mean())
 train_df['Age*Class'] = train_df['Age*Class'].fillna(train_df['Age*Class'].mean())
 test_df['Age'] = test_df['Age'].fillna(test_df['Age'].mean())
 test_df['Age*Class'] = test_df['Age*Class'].fillna(test_df['Age*Class'].mean())
 train_x = train_df.copy()
test_x = test_df.copy()
train_nn = pd.read_csv('.. /input/titanic/train.csv')
 train_x_nn = train_df.copy()
train_y_nn = train_nn['Survived']
test_x_nn = pd.read_csv('.. /input/titanic/test.csv')
 test_x_nn = test_df.copy()
base_param = {
'input_dropout': 0.05,
'hidden_layers': 2.0,
'hidden_units': 96.0,
'hidden_activation': 'prelu',
'hidden_dropout': 0.05,
'batch_norm': 'before_act',
'optimizer': {'type': 'adam', 'lr': 0.00037640141509672924},
'batch_size': 32.0}
model_1a = Model1Xgb()
pred_train_1a, pred_test_1a = predict_cv(model_1a, train_x, train_y, test_x)
 model_1b = Model1NN(base_param)
 pred_train_1b, pred_test_1b = predict_cv(model_1b, train_x_nn, train_y_nn, test_x_nn)
 print(f'logloss: {log_loss(train_y, pred_train_1a, eps=1e-7):.4f}')
 print(f'logloss: {log_loss(train_y, pred_train_1b, eps=1e-7):.4f}')
 train_x_2 = pd.DataFrame({'pred_1a': pred_train_1a, 'pred_1b': pred_train_1b})
 test_x_2 = pd.DataFrame({'pred_1a': pred_test_1a, 'pred_1b': pred_test_1b})
 model_2 = Model2Linear()
pred_train_2, pred_test_2 = predict_cv(model_2, train_x_2, train_y, test_x_2)
 print(f'logloss: {log_loss(train_y, pred_train_2, eps=1e-7):.4f}')
 pred_test_2 = np.where(pred_test_2 < 0.5, 0, 1)
 pred_test_2 = pred_test_2.tolist()
print(pred_test_2)
 sub = pd.read_csv('.. /input/titanic/gender_submission.csv')
 sub['Survived'] = list(map(int, pred_test_2))
sub.to_csv('submission_stacking3.csv', index=False)
<load_from_csv>
tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')
 tourney_seed = tourney_seed[tourney_seed['Season'] > 2014]
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )<train_model>
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')
 season_result = season_result[season_result['Season'] > 2014]
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
data_size_train = train.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_train )<train_model>
test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1 )
data_size_test = test.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_test )<categorify>
test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))
test_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df
train = train.drop(["PoolQC", "MiscFeature", "Alley", "Fence", "Street", "Alley"], axis = 1)
 none_values_train = ["FireplaceQu", "GarageCond", "GarageType", "GarageFinish", "GarageQual",
"BsmtExposure", "BsmtFinType2", "BsmtFinType1", "BsmtCond", "BsmtQual",
"MasVnrType", "Electrical"]
for value in none_values_train:
train = train.fillna({value : "None"})
 zero_values_train = ["LotFrontage", "GarageYrBlt", "MasVnrArea"]
for value in zero_values_train:
train = train.fillna({value : 0} )<categorify>
X = train_df.drop('result', axis=1)
 y = train_df.result
test = test.drop(["PoolQC", "MiscFeature", "Alley", "Fence", "Street", "Alley"], axis = 1)
 none_values_test = ["FireplaceQu", "GarageCond", "GarageType", "GarageFinish", "GarageQual",
"BsmtExposure", "BsmtFinType2", "BsmtFinType1", "BsmtCond", "BsmtQual",
"MasVnrType", "Electrical", "MSZoning", "Utilities", "Functional",
"Exterior2nd", "SaleType", "Exterior1st", "KitchenQual"]
for value in none_values_test:
test = test.fillna({value : "None"})
 zero_values_test = ["LotFrontage", "GarageYrBlt", "MasVnrArea", "BsmtHalfBath", "BsmtFullBath",
"BsmtFinSF2", "BsmtFinSF1", "TotalBsmtSF", "GarageArea", "GarageCars", "BsmtUnfSF"]
for value in zero_values_test:
test = test.fillna({value : 0} )<drop_column>
params_lgb = {'num_leaves': lgb_num_leaves_max,
'min_data_in_leaf': lgb_in_leaf,
'objective': 'binary',
'max_depth': -1,
'learning_rate': lgb_lr,
"boosting_type": "gbdt",
"bagging_seed": lgb_bagging,
"metric": 'logloss',
"verbosity": -1,
'random_state': 42,
}
train = train.drop(train[train["GrLivArea"] > 4000].index )<data_type_conversions>
NFOLDS = 10
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_lgb = np.zeros(test_df.shape[0])
 y_train_lgb = np.zeros(X.shape[0])
 y_oof = np.zeros(X.shape[0])
 feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
y_train_lgb += clf.predict(X)/ NFOLDS
y_preds_lgb += clf.predict(test_df)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
upper = corr.where(np.triu(np.ones(corr.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.90)]
print(to_drop )<import_modules>
params_xgb = {'max_depth': xgb_max_depth,
'objective': 'binary:logistic',
'min_child_weight': xgb_min_child_weight,
'learning_rate': xgb_lr,
'eta' : 0.3,
'subsample': 0.8,
'lambda ' : 4,
'eval_metric': 'logloss',
'colsample_bytree ': 0.9,
'colsample_bylevel': 1
}
from sklearn.linear_model import Ridge, Lasso, LinearRegression
from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, StackingRegressor
from sklearn.model_selection import GridSearchCV, train_test_split, cross_val_score, learning_curve
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
import lightgbm as lgb<prepare_x_and_y>
NFOLDS = 10
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_xgb = np.zeros(test_df.shape[0])
 y_train_xgb = np.zeros(X.shape[0])
 y_oof_xgb = np.zeros(X.shape[0])
 train_df_set = xgb.DMatrix(X)
 test_set = xgb.DMatrix(test_df)
 for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_set = xgb.DMatrix(X_train, y_train)
 val_set = xgb.DMatrix(X_valid, y_valid)
 clf = xgb.train(params_xgb, train_set, num_boost_round=xgb_num_boost_round_max, evals=[(train_set, 'train'),(val_set, 'val')], verbose_eval=100)
 y_train_xgb += clf.predict(train_df_set)/ NFOLDS
y_preds_xgb += clf.predict(test_set)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
all_data = pd.concat([train.drop("SalePrice", axis = 1), test])
 all_data = pd.get_dummies(all_data)
 train_dummy = all_data[0:1456]
test_dummy = all_data[1456:2915]
train_dummy["SalePrice"] = train["SalePrice"]
X = train_dummy.drop(["SalePrice", "Id"], axis = 1 ).values
Y = train_dummy["SalePrice"].values<normalization>
%%time
scaler = StandardScaler()
train_log = pd.DataFrame(
 scaler.fit_transform(X),
columns=X.columns,
index=X.index 
)
 test_log = pd.DataFrame(
 scaler.transform(test_df),
columns=test_df.columns,
index=test_df.index
)
scaler = StandardScaler()
X = scaler.fit_transform(X)
 test_dummy = test_dummy.drop("Id", axis = 1)
 test_dummy = scaler.fit_transform(test_dummy )<choose_model_class>
logreg = LogisticRegression()
logreg.fit(train_log, y)
 coeff_logreg = pd.DataFrame(train_log.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 coeff_logreg.sort_values(by='score_logreg', ascending=False )
pipe_cat = Pipeline([ 
("scaler", StandardScaler()), 
("cat", CatBoostRegressor(verbose = 0)) ])
 params_cat = {
"cat__depth" : [3, 4, 5],
"cat__learning_rate" : [0.001, 0.01, 0.1, 1],
"cat__n_estimators" : [300, 400, 500]}
model_cat = CatBoostRegressor(verbose = 0,
depth = 5,
learning_rate = 0.1,
n_estimators = 500)
 model_cat.fit(X, Y)
 model_score = cross_val_score(model_cat, X, Y, cv = 5, n_jobs = -1)
 Y_preds = model_cat.predict(X)
 print("R2 : " + str(model_score.mean()))
print("RMSE : " + str(np.sqrt(mean_squared_error(Y, Y_preds))))<choose_model_class>
eli5.show_weights(logreg )
pipe_gbm = Pipeline([ 
("gbm", lgb.LGBMRegressor())])
 params_gbm = {
"gbm__n_estimators" : [300, 500, 700],
"gbm__learning_rate" : [0.001, 0.01, 0.1],
"gbm__max_depth" : [3, 6, 9],
"gbm__max_bin" : [32, 64, 128, 256, 512]}
model_gbm = lgb.LGBMRegressor(verbose = 0,
learning_rate = 0.1,
max_bin = 32,
max_depth = 4,
n_estimators = 500)
 model_gbm.fit(X, Y)
 model_score = cross_val_score(model_gbm, X, Y, cv = 5, n_jobs = -1)
 Y_preds = model_gbm.predict(X)
 print("R2 : " + str(model_score.mean()))
print("RMSE : " + str(np.sqrt(mean_squared_error(Y, Y_preds))))<find_best_model_class>
y_logreg_train = logreg.predict(train_log)
 y_logreg_pred = logreg.predict(test_log )
estimators = [("CatBoost", model_cat), 
("LightGBM", model_gbm)]
stacker = StackingRegressor(estimators = estimators)
 stacker.fit(X, Y)
 stacker_score = cross_val_score(stacker, X, Y, cv = 5, n_jobs = -1)
 Y_pred_stack = stacker.predict(X)
 Y_preds_sub = stacker.predict(test_dummy)
 print("R2 : " + str(stacker_score.mean()))
print("RMSE : " + str(np.sqrt(mean_squared_error(Y, Y_pred_stack))))<save_to_csv>
def plot_cm(y_true, y_pred, title, figsize=(7,6)) :
y_pred = y_pred.round().astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )
submission = pd.DataFrame({"Id" : test.Id, "SalePrice" : Y_preds_sub})
 submission.to_csv("submission_reg.csv", index = False)
 submission.head()<save_to_csv>
y_preds = w_lgb*y_preds_lgb + w_xgb*y_preds_xgb + w_logreg*y_logreg_pred
f = open(".. /input/submission/final_5_0.10117320406891336.csv")
 tem = pd.read_csv(f)
 tem.head(10)
 tem.to_csv('submission.csv', index=False )<import_modules>
sub['Pred'] = y_preds
sub.head()
import pandas as pd
import numpy as np
import seaborn as sns
import pylab as pl
import matplotlib.pyplot as plt
import math
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score<set_options>
sub.to_csv('submission.csv', index=False )
<load_from_csv><EOS> 
sub.to_csv('submission.csv', index=False )