Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<import_modules>
plt.rcParams['font.size'] = 22
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd 
<load_from_csv>
train_bureau = pd.read_csv('.. /input/home-credit-manual-engineered-features/train_bureau_raw.csv', nrows = 1000)
 test_bureau = pd.read_csv('.. /input/home-credit-manual-engineered-features/test_bureau_raw.csv', nrows = 1000)
 train_previous = pd.read_csv('.. /input/home-credit-manual-engineered-features/train_previous_raw.csv', nrows = 1000)
 test_previous = pd.read_csv('.. /input/home-credit-manual-engineered-features/test_previous_raw.csv', nrows = 1000)
 bureau_columns = list(train_bureau.columns)
 previous_columns = list(train_previous.columns )
df = pd.read_csv(".. /input/pk-hska/agegroups_train.csv")
 df.head()<define_variables>
bureau_features = list(set(bureau_columns)- set(previous_columns))
previous_features = list(set(previous_columns)- set(bureau_columns))
original_features = list(set(previous_columns)& set(bureau_columns))
print('There are %d original features.' % len(original_features))
print('There are %d bureau and bureau balance features.' % len(bureau_features))
print('There are %d previous Home Credit loan features.' % len(previous_features))
features = ['weekday_number', 'region_number', 'sum_sales_pg1', 'sum_sales_pg2', 'sum_sales_pg3']
target = 'age_group_number'<prepare_x_and_y>
train_labels = train_bureau['TARGET']
previous_features.append('SK_ID_CURR')
 train_ids = train_bureau['SK_ID_CURR']
test_ids = test_bureau['SK_ID_CURR']
train = train_bureau.merge(train_previous[previous_features], on = 'SK_ID_CURR')
 test = test_bureau.merge(test_previous[previous_features], on = 'SK_ID_CURR')
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
X = df[features]
y = df[target]<train_on_grid>
train = pd.get_dummies(train)
 test = pd.get_dummies(test)
 train, test = train.align(test, join = 'inner', axis = 1)
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
clf = RandomForestClassifier()
param_grid = [
{'max_depth': range(5, 21, 5),
'n_estimators' : range(50, 101, 50)}
]
search = GridSearchCV(clf, param_grid, cv=3)
 search.fit(X, y)
 print("Best parameter(CV score={:.2f}):{})".format(search.best_score_, search.best_params_))<load_from_csv>
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper.head()
df_test = pd.read_csv(".. /input/pk-hska/agegroups_test.csv" ).set_index('index')
 df_test.head()<predict_on_test>
to_drop = [column for column in upper.columns if any(upper[column] > threshold)]
print('There are %d columns to remove.' %(len(to_drop)) )
prediction = search.predict(df_test)
 prediction<feature_engineering>
train = train.drop(columns = to_drop)
 test = test.drop(columns = to_drop)
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
df_test['age_group_number'] = prediction<save_to_csv>
train = pd.read_csv('.. /input/home-credit-manual-engineered-features/m_train_combined.csv')
 test = pd.read_csv('.. /input/home-credit-manual-engineered-features/m_test_combined.csv' )
df_submission = df_test[['age_group_number']]
df_submission.to_csv('submission.csv', index=True, index_label='index' )<import_modules>
train_missing =(train.isnull().sum() / len(train)).sort_values(ascending = False)
 train_missing.head()
import numpy as np
import pandas as pd<install_modules>
test_missing =(test.isnull().sum() / len(test)).sort_values(ascending = False)
 test_missing.head()
!pip install "torch==1.4" "torchvision==0.5.0"
torch.__version__<import_modules>
train_missing = train_missing.index[train_missing > 0.75]
test_missing = test_missing.index[test_missing > 0.75]
all_missing = list(set(set(train_missing)| set(test_missing)))
 print('There are %d columns with more than 75%% missing values' % len(all_missing))
from fastai.vision import *
from fastai.metrics import error_rate<set_options>
train_labels = train["TARGET"]
train_ids = train['SK_ID_CURR']
test_ids = test['SK_ID_CURR']
train = pd.get_dummies(train.drop(columns = all_missing))
test = pd.get_dummies(test.drop(columns = all_missing))
train, test = train.align(test, join = 'inner', axis = 1)
 print('Training set full shape: ', train.shape)
 print('Testing set full shape: ' , test.shape )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<define_variables>
train = train.drop(columns = ['SK_ID_CURR'])
 test = test.drop(columns = ['SK_ID_CURR'] )
copyfile(src = ".. /input/cutmix/shared/0_image_data_augmentation/exp/nb_new_data_augmentation.py" ,
dst = ".. /working/nb_new_data_augmentation.py" )<import_modules>
feature_importances = np.zeros(train.shape[1])
 model = lgb.LGBMClassifier(objective='binary', boosting_type = 'goss', n_estimators = 10000, class_weight = 'balanced' )
from nb_new_data_augmentation import *<define_variables>
for i in range(2):
train_features, valid_features, train_y, valid_y = train_test_split(train, train_labels, test_size = 0.25, random_state = i)
 model.fit(train_features, train_y, early_stopping_rounds=100, eval_set = [(valid_features, valid_y)],
eval_metric = 'auc', verbose = 200)
 feature_importances += model.feature_importances_
path = Path('.. /input/shopee-product-detection-student' )<feature_engineering>
zero_features = list(feature_importances[feature_importances['importance'] == 0.0]['feature'])
 print('There are %d features with 0.0 importance' % len(zero_features))
feature_importances.tail()
N = 1000
df = pd.read_csv('.. /input/shopee-product-detection-student/train.csv')
 df = df.groupby('category' ).apply(lambda x: x[:N] ).reset_index(drop=True)
 df['filename'] = 'train/' + df['category'].apply(lambda x: '{0:0>2}'.format(x)).apply(str)+'/'+df['filename']<categorify>
train = train.drop(columns = zero_features)
 test = test.drop(columns = zero_features)
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
df['category'] = df['category'].replace({0: '00 Dress',1: '01 Sarung',2: '02 Shirt(Top)',3: '03 Long Sleeves/Hoodie',4: '04 Jeans(Female)',5: '05 Ring',6: '06 Ear Rings',7: '07 Cap',8: '08 Purse/Wallet',9: '09 Bags',10: '10 Phone Cover',11: '11 Phone',12: '12 Clock',13: '13 Plastic Baby Bottles',14: '14 Rice Cooker',15: '15 Coffee',16: '16 Shoe',17: '17 High Heels',18: '18 Aircon',19: '19 Pendrive',20: '20 Chair',21: '21 Racket',22: '22 Helmet',23: '23 Gloves',24: '24 Watch',25: '25 Belt',26: '26 Headphones/Earpiece',27: '27 Toy Car',28: '28 Suit/Jacket(Male)',29: '29 Tuxedo Pants(Male)',30: '30 Sport Shoes',31: '31 Biscuit/ Junk Food',32: '32 Face Mask',33: '33 Sanitizer/Antiseptic',34: '34 Skin Care(?)',35: '35 Perfume/Cologne',36: '36 Cleaning Supplies',37: '37 Laptop',38: '38 Bowls',39: '39 Vases?',40: '40 Shower Stuff(Showerhead)',41: '41 Sofa',})
 df<load_pretrained>
def identify_zero_importance_features(train, train_labels, iterations = 2):

 feature_importances = np.zeros(train.shape[1])
 model = lgb.LGBMClassifier(objective='binary', boosting_type = 'goss', n_estimators = 10000, class_weight = 'balanced')
 for i in range(iterations):
train_features, valid_features, train_y, valid_y = train_test_split(train, train_labels, test_size = 0.25, random_state = i)
 model.fit(train_features, train_y, early_stopping_rounds=100, eval_set = [(valid_features, valid_y)],
eval_metric = 'auc', verbose = 200)
 feature_importances += model.feature_importances_ / iterations
feature_importances = pd.DataFrame({'feature': list(train.columns), 'importance': feature_importances} ).sort_values('importance', ascending = False)
 zero_features = list(feature_importances[feature_importances['importance'] == 0.0]['feature'])
 print('
There are %d features with 0.0 importance' % len(zero_features))
return zero_features, feature_importances
np.random.seed(0)
 data = ImageDataBunch.from_df(path, df, folder='/train/train',valid_pct=0.2,
test='test/test/test',
ds_tfms=get_transforms() ,
size=224, bs=64
).normalize(imagenet_stats )<define_variables>
second_round_zero_features, feature_importances = identify_zero_importance_features(train, train_labels )
data.show_batch(rows=3, figsize=(10, 5))<choose_model_class>
threshold = 0.95
features_to_keep = list(norm_feature_importances[norm_feature_importances['cumulative_importance'] < threshold]['feature'])
 train_small = train[features_to_keep]
test_small = test[features_to_keep]
learn = cnn_learner(data, models.resnet50,
metrics=[error_rate, accuracy]
).cutmix().show_multi_img_tfms()<define_variables>
train_small['TARGET'] = train_labels
train_small['SK_ID_CURR'] = train_ids
test_small['SK_ID_CURR'] = test_ids
train_small.to_csv('m_train_small.csv', index = False)
 test_small.to_csv('m_test_small.csv', index = False )
learn.model_dir = '/kaggle/working/'<find_best_params>
def model(features, test_features, encoding = 'ohe', n_folds = 5):

 train_ids = features['SK_ID_CURR']
test_ids = test_features['SK_ID_CURR']
labels = features['TARGET']
features = features.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_features.drop(columns = ['SK_ID_CURR'])
 if encoding == 'ohe':
features = pd.get_dummies(features)
 test_features = pd.get_dummies(test_features)
 features, test_features = features.align(test_features, join = 'inner', axis = 1)
 cat_indices = 'auto'
elif encoding == 'le':
label_encoder = LabelEncoder()
cat_indices = []
for i, col in enumerate(features):
if features[col].dtype == 'object':
features[col] = label_encoder.fit_transform(np.array(features[col].astype(str)).reshape(( -1,)))
 test_features[col] = label_encoder.transform(np.array(test_features[col].astype(str)).reshape(( -1,)))
 cat_indices.append(i)
 else:
raise ValueError("Encoding must be either 'ohe' or 'le'")
 print('Training Data Shape: ', features.shape)
 print('Testing Data Shape: ', test_features.shape)
 feature_names = list(features.columns)
 features = np.array(features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = False, random_state = 50)
 feature_importance_values = np.zeros(len(feature_names))
test_predictions = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(features):
train_features, train_labels = features[train_indices], labels[train_indices]
valid_features, valid_labels = features[valid_indices], labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary', boosting_type='goss',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1, n_jobs = -1, random_state = 50)
 model.fit(train_features, train_labels, eval_metric = 'auc',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'], categorical_feature = cat_indices,
early_stopping_rounds = 100, verbose = 200)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_predictions += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(valid_features, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, train_features, valid_features
gc.collect()
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_predictions})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return submission, feature_importances, metrics
learn.lr_find()
learn.recorder.plot(suggestion=True )<train_model>
train['TARGET'] = train_labels
train['SK_ID_CURR'] = train_ids
test['SK_ID_CURR'] = test_ids
submission, feature_importances, metrics = model(train, test )
learn.fit_one_cycle(8, max_lr=slice(2e-3))<train_model>
submission.to_csv('selected_features_submission.csv', index = False )
learn.fit_one_cycle(8, max_lr=slice(1e-5 , 4e-4))<choose_model_class>
submission_small, feature_importances_small, metrics_small = model(train_small, test_small )
interp = ClassificationInterpretation.from_learner(learn )<set_options>
submission_small.to_csv('selected_features_small_submission.csv', index = False )
interp.most_confused(min_val=40 )<predict_on_test>
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 pipeline = Pipeline(steps = [('imputer', Imputer(strategy = 'median')) , 
('pca', PCA())])
 train_pca = pipeline.fit_transform(train)
 test_pca = pipeline.transform(test )
<define_variables><EOS> 
print('2 principal components account for {:.4f}% of the variance.'.format(100 * np.sum(pca.explained_variance_ratio_[:2])) )