Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<set_options>
%matplotlib inline
data_path=".. /input"
%matplotlib inline<load_from_csv>
data = pd.concat([pd.read_csv(os.path.join(data_path, "application_train.csv"), index_col="SK_ID_CURR"), pd.read_csv(os.path.join(data_path, "application_test.csv"), index_col="SK_ID_CURR")], axis=0)
 data.head()
df = pd.read_csv('.. /input/train.csv')
 df.head()
train, validate = np.split(df, [int (.8*len(df)) ])
 validate.index = range(len(validate))<categorify>
data.TARGET.fillna(-1 ).value_counts()
class ImageDataset(Dataset):
def __init__(self, data, root_dir, transform=None):
self.df = data
self.dir = root_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_path = os.path.join(self.dir, self.df['Id'][index])
 image = Image.open(img_path ).convert('RGB')
 label = np.array(self.df['Category'][index])
 if self.transform:
image = self.transform(image)
 sample =(image, label)
 return sample 
<normalization>
for c in data.select_dtypes(include="object"):
data[c] = data[c].astype("category" )
transform = transforms.Compose([transforms.Resize(256),
transforms.RandomRotation(24),
transforms.ColorJitter(brightness=0.8, contrast=0.2),
transforms.CenterCrop(224),
transforms.RandomHorizontalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 trainset = ImageDataset(data=train, root_dir='.. /input/data/data', transform=transform)
 valset = ImageDataset(data=validate, root_dir='.. /input/data/data', transform=transform)
 datasets = {'train' : trainset, 'val': valset}
dataloaders = {x : DataLoader(datasets[x], batch_size=32, num_workers=4, shuffle=True)
 for x in ['train', 'val']} 
<train_model>
data.dtypes.value_counts()
gpu = torch.cuda.is_available()
if gpu:
print("CUDA is available.Training on GPU")
 else:
print("CUDA unavailable.Training on CPU" )<choose_model_class>
bureau = pd.read_csv(os.path.join(data_path, "bureau.csv"))
bureau.head()
model = models.resnet152(pretrained=True)
 num_ft = model.fc.in_features
model.fc = nn.Sequential(nn.Linear(num_ft, 67),
nn.Dropout2d(p=0.2))
model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters() , lr=0.001, momentum=0.9)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1 )<train_model>
bureau_balance = pd.read_csv(os.path.join(data_path, "bureau_balance.csv")).join(bureau.set_index("SK_ID_BUREAU" ).SK_ID_CURR, on="SK_ID_BUREAU" )
def train_model(model, criterion, optimizer, scheduler, num_epochs=20):
since = time.time()
best_acc = 0.0
best_model_state = copy.deepcopy(model.state_dict())
 for epoch in range(num_epochs):
epoch += 1
print("Epoch : {}/{}".format(epoch, num_epochs))
print("-"*10)
 for phase in ['train', 'val']:
if phase is 'train':
scheduler.step()
model.train()
else:
model.eval()
running_loss = 0.0
running_correct = 0
for inputs, labels in dataloaders[phase]:
if gpu:
inputs = Variable(inputs.cuda())
 labels = Variable(labels.cuda())
 else:
inputs = Variable(inputs)
 labels = Variable(labels)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase is 'train'):
outputs = model(inputs)
 _, predicted = torch.max(outputs.data, 1)
 loss = criterion(outputs, labels)
 if phase is 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 running_correct += torch.sum(predicted.cpu() == labels.cpu())
 epoch_loss = running_loss / len(datasets[phase])
 epoch_acc = running_correct.double() / len(datasets[phase])
 print("{}: Loss: {:.4f}, Accuracy: {:.4f}".format(phase, epoch_loss, epoch_acc))
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_state = copy.deepcopy(model.state_dict())
 print()
time_elapsed = time.time() - since
print("Training finished in {} seconds!".format(time_elapsed))
print("Best eval accuracy: {:.4f}".format(100*best_acc))
model.load_state_dict(best_model_state)
 return model<train_model>
bureau_balance_dumies = pd.get_dummies(bureau_balance.STATUS)
 bureau_balance_dumies["SK_ID_CURR"] = bureau_balance.SK_ID_CURR
bureau_balance_dumies = bureau_balance_dumies.groupby("SK_ID_CURR" ).sum()
model = train_model(model, criterion, optimizer, scheduler, 25 )<load_from_csv>
irregular = ["bureau_balance_status_" + c for c in bureau_balance_dumies if c not in ["0", "C", "X"]]
very_irregular = ["bureau_balance_status_" + c for c in bureau_balance_dumies if c not in ["0", "1", "2", "C", "X"]]
testset = ImageDataset(data=pd.read_csv('.. /input/sample_sub.csv'), root_dir='.. /input/data/data/', transform=transform)
 testloader = DataLoader(testset, batch_size=32, num_workers=4, shuffle=False )<load_from_csv>
bureau_balance_dumies.columns = ["bureau_balance_status_" + c for c in bureau_balance_dumies]
submission = pd.read_csv('.. /input/sample_sub.csv')
 submission.head()<categorify>
bureau_balance_dumies["total_balances"] = bureau_balance_dumies.sum(axis=1)
 bureau_balance_dumies["irregular_bureau_balance"] = bureau_balance_dumies[irregular].sum(axis=1)
 bureau_balance_dumies["irregular_bureau_balance_ratio"] = bureau_balance_dumies["irregular_bureau_balance"] / bureau_balance_dumies["total_balances"]
bureau_balance_dumies["very_irregular_bureau_balance"] = bureau_balance_dumies[very_irregular].sum(axis=1)
 bureau_balance_dumies["very_irregular_bureau_balance_ratio"] = bureau_balance_dumies["very_irregular_bureau_balance"] / bureau_balance_dumies["total_balances"]
bureau_balance_dumies["super_irregular_bureau_balance_ratio"] = bureau_balance_dumies["bureau_balance_status_5"] / bureau_balance_dumies["total_balances"]
bureau_balance_dumies.head(100 )
predictions = []
for inputs, labels in testloader:
if gpu:
inputs = Variable(inputs.cuda())
 else:
inputs = Variable(inputs)
 outputs = model(inputs)
 _, predicted = torch.max(outputs.data, 1)
 for i in range(len(predicted)) :
predictions.append(int(predicted[i]))
submission['Category'] = predictions<save_to_csv>
extended_data = [bureau_balance_dumies]
del bureau_balance_dumies, bureau_balance
submission.to_csv('Submission5.csv', index=False, encoding='utf-8' )<import_modules>
for c in bureau.select_dtypes(["object"]):
display(bureau[c].value_counts(dropna=False ).to_frame() )
import warnings
import numpy as np
import pandas as pd
from pathlib import Path
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from catboost import CatBoostClassifier<define_variables>
bureau.groupby("SK_ID_CURR" ).CREDIT_CURRENCY.unique().apply(len ).value_counts()
PATH_TO_DATA = Path('.. /input/focus-start-2020' )<load_from_csv>
credits_status = pd.get_dummies(bureau.CREDIT_ACTIVE)
 credits_status["SK_ID_CURR"] = bureau.SK_ID_CURR
credits_status = credits_status.groupby("SK_ID_CURR" ).sum(axis=0)
 credits_status.columns = ["credits_status_" + "_".join(c.lower().split())for c in credits_status]
total_credits = credits_status.sum(axis=1)
 credits_status["sold_or_bad_credits"] = credits_status[["credits_status_bad_debt", "credits_status_sold"]].sum(axis=1)
 ratios = pd.concat(((credits_status[c] / total_credits ).rename(c + "_ratio")for c in credits_status), axis=1)
 credits_status["total_credits"] = total_credits
credits_status = credits_status.join(ratios)
 credits_status
train_df = pd.read_csv(PATH_TO_DATA / 'flight_delays_train.csv' )<load_from_csv>
extended_data.append(credits_status)
 del credits_status
test_df = pd.read_csv(PATH_TO_DATA / 'flight_delays_test.csv' )<feature_engineering>
credits_types = pd.get_dummies(bureau.CREDIT_TYPE)
 credits_types["SK_ID_CURR"] = bureau.SK_ID_CURR
credits_types = credits_types.groupby("SK_ID_CURR" ).sum(axis=0)
 credits_types.columns = ["credits_types_" + "_".join(c.lower().split())for c in credits_types]
credits_types
def transform(df):
df['flight'] = df['origin'] + '-->' + df['dest']
df['Month'] = df['month']
df['DayofMonth'] = df['day_of_month']
df['DayOfWeek'] = df['day_of_week']
df['DepTime'] = df['dep_time']
df['DepTime_hour'] = df['DepTime'] // 100
df['DepTime_minute'] = df['DepTime'] % 100
df['morning'] =(( df['DepTime_hour'] >= 6)&(df['DepTime_hour'] < 12))
df['day'] =(( df['DepTime_hour'] >= 12)&(df['DepTime_hour'] < 18))
df['evening'] =(( df['DepTime_hour'] >= 18)&(df['DepTime_hour'] < 24))
df['night'] =(( df['DepTime_hour'] >= 0)&(df['DepTime_hour'] < 6))
df['low_delay'] =(( df['DepTime_hour'] >= 4)&(df['DepTime_hour'] < 9))
df['other_time'] =(( df['DepTime_hour'] < 9)&(df['DepTime_hour'] >=5))
df['delay_time'] =(( df['DepTime_hour'] >= 13)&(df['DepTime_hour'] < 24)|(df['DepTime_hour'] < 5))
df['middle_time'] =(( df['DepTime_hour'] >= 9)&(df['DepTime_hour'] < 13))
df['high_delay'] =(( df['DepTime_hour'] >= 4)&(df['DepTime_hour'] < 9))
df['flight_T1'] = df['dest'] + '-->' + df['DepTime'].astype("str")
 df['flight_T2'] = df['origin'] + '-->' + df['DepTime'].astype("str")
 df['flight_T2_hour'] = df['origin'] + '-->' + df['DepTime_hour'].astype("str")
 df['flight_T2_min'] = df['origin'] + '-->' + df['DepTime_minute'].astype("str")
 df['flight_T3_hour'] = df['dest'] + '-->' + df['DepTime_hour'].astype("str")
 df['flight_T3_min'] = df['dest'] + '-->' + df['DepTime_minute'].astype("str")
 df['fl_Un'] = df['flight'] + '-->' + df['unique_carrier'].astype("str")
 df['Or_Un'] = df['origin'] + '-->' + df['unique_carrier'].astype("str")
 df['Dest_Un'] = df['dest'] + '-->' + df['unique_carrier'].astype("str")
 df['flight_M1'] = df['dest'].astype(str)+ '-->' + df['Month'].astype(str)
 df['flight_D1'] = df['dest'].astype(str)+ '-->' + df['DayOfWeek'].astype(str)
 df['flight_M2'] = df['dest'].astype(str)+ '-->' + df['DayofMonth'].astype(str)
 return df<categorify>
extended_data.append(credits_types)
 del credits_types
train_df = transform(train_df)
 test_df = transform(test_df )<drop_column>
bureau.DAYS_ENDDATE_FACT.fillna(bureau.DAYS_CREDIT_ENDDATE, inplace=True)
 bureau.DAYS_CREDIT_ENDDATE.fillna(bureau.DAYS_ENDDATE_FACT, inplace=True)
 bureau.DAYS_ENDDATE_FACT.fillna(0, inplace=True)
 bureau.DAYS_CREDIT_ENDDATE.fillna(0, inplace=True)
 bureau["days_end_credit_advancement"] = bureau.DAYS_CREDIT_ENDDATE - bureau.DAYS_ENDDATE_FACT
max_cols = ["CREDIT_DAY_OVERDUE", "AMT_CREDIT_MAX_OVERDUE", "CNT_CREDIT_PROLONG", "days_end_credit_advancement"]
bureau_agg_data = bureau[max_cols + ["SK_ID_CURR"]].fillna(0 ).groupby("SK_ID_CURR" ).max()
bureau_agg_data.columns = ["max_" + "_".join(c.lower().split())for c in bureau_agg_data]
display(bureau_agg_data)
categ_feat_idx = np.where(train_df.drop('dep_delayed_15min', axis=1 ).dtypes == 'object')[0]
categ_feat_idx<prepare_x_and_y>
extended_data.append(bureau_agg_data)
 del bureau_agg_data
X_train = train_df.drop('dep_delayed_15min', axis=1 ).values
y_train = train_df['dep_delayed_15min']
X_test = test_df.values<split>
sum_cols = ["CREDIT_DAY_OVERDUE", "AMT_CREDIT_MAX_OVERDUE", "CNT_CREDIT_PROLONG", "days_end_credit_advancement", "AMT_CREDIT_SUM", "AMT_CREDIT_SUM_DEBT", "AMT_CREDIT_SUM_LIMIT", "AMT_CREDIT_SUM_OVERDUE"]
bureau_agg_data = bureau[sum_cols + ["SK_ID_CURR"]].fillna(0 ).groupby("SK_ID_CURR" ).sum(axis=0)
 bureau_agg_data["total_outstanding_debt_ratio"] = bureau_agg_data.AMT_CREDIT_SUM_DEBT / bureau_agg_data.AMT_CREDIT_SUM
bureau_agg_data.columns = ["sum_" + "_".join(c.lower().split())for c in bureau_agg_data]
display(bureau_agg_data)
X_train_part, X_valid, y_train_part, y_valid = train_test_split(X_train, y_train,
test_size=0.3,
random_state=42 )<choose_model_class>
extended_data.append(bureau_agg_data)
 del bureau_agg_data
ctb = CatBoostClassifier(random_seed=42, silent=True )<train_model>
sum_cols = ["AMT_CREDIT_SUM", "AMT_CREDIT_SUM_DEBT"]
active_bureau = bureau[bureau.CREDIT_ACTIVE == "Active"][sum_cols + ["SK_ID_CURR"]].groupby("SK_ID_CURR" ).sum(axis=0)
 active_bureau.columns = ["total_active_debts", "outstanding_active_debts"]
display(active_bureau)
 extended_data.append(active_bureau)
 del active_bureau
%%time
ctb.fit(X_train_part, y_train_part, cat_features=categ_feat_idx )<predict_on_test>
data = data.join(pd.concat(extended_data, axis=1))
display(data)
 del extended_data
ctb_valid_pred = ctb.predict_proba(X_valid)[:, 1]
print(roc_auc_score(y_valid, ctb_valid_pred))<train_model>
data["credit_goodsprice_ratio"] = data.AMT_CREDIT / data.AMT_GOODS_PRICE
data["annuity_income_ratio"] = data.AMT_ANNUITY / data.AMT_INCOME_TOTAL
data["annuity_credit_ratio"] = data.AMT_ANNUITY / data.AMT_CREDIT
data["total_new_credit_increase"] = data.AMT_CREDIT + data.outstanding_active_debts
data["credit_increase"] = data.AMT_CREDIT / data.total_new_credit_increase
data["total_new_credit_income_ratio"] = data.total_new_credit_increase / data.AMT_INCOME_TOTAL
%%time
ctb.fit(X_train, y_train,
cat_features=categ_feat_idx);<predict_on_test>
def train_model(train, target, nl, X_test, folds=5):
test_probs = []
for i in range(folds):
X_valid = train.sample(frac=1/folds)
 y_valid = X_valid[target]
X_valid = X_valid.drop(target, axis=1)
 X_train = train.drop(X_valid.index)
 y_train = X_train[target]
X_train = X_train.drop(target, axis=1)
 learner = LGBMClassifier(n_estimators=10000, num_leaves=nl)
 learner.fit(X_train, y_train, early_stopping_rounds=10, eval_metric="auc", verbose=50,
eval_set=[(X_train, y_train), 
(X_valid, y_valid)])
 probs = pd.Series(learner.predict_proba(X_test)[:, -1], index=X_test.index, name="fold_" + str(i))
test_probs.append(probs)
 return pd.concat(test_probs, axis=1 ).mean(axis=1 )
ctb_test_pred = ctb.predict_proba(X_test)[:, 1]<load_from_csv>
test = data[data.TARGET.notnull() ].sample(frac=0.1)
 train = data[data.TARGET.notnull() ].drop(test.index)
 X_test = test.drop("TARGET", axis=1)
 y_test = test.TARGET
nls = [2 ** i for i in [4, 5, 6]]
res = pd.Series([np.nan] * len(nls), index=nls, name="ROC_AUC")
 for nl in nls:
print("*"*10, nl, "*"*10)
 probs = train_model(train, "TARGET", nl, X_test, 10)
 res.loc[nl] = roc_auc_score(y_test, probs)
 print("ROC_AUC para {nl} hojas: {res:.4f}".format(nl=nl, res=res.loc[nl]))
<save_to_csv><EOS> 
train = data[data.TARGET.notnull() ]
X_test = data.drop(train.index ).drop("TARGET", axis=1)
 nl = res.idxmax()
train_model(train, "TARGET", nl, X_test ).rename("TARGET" ).to_csv("submission_{nl}.csv".format(nl=nl), header=True )