Kaggle competition: Home Credit Default Risk
test_scaled=scaler.fit_transform(test_ )<predict_on_test>
import gc
y_pred=vote.predict(test_scaled )<create_dataframe>
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
sub=pd.DataFrame()
sub['id']=test['id']
sub['heart_disease']=y_pred 
<save_to_csv>
data = pd.read_csv(".. /input/application_train.csv")
 test = pd.read_csv('.. /input/application_test.csv')
 prev = pd.read_csv('.. /input/previous_application.csv')
 buro = pd.read_csv('.. /input/bureau.csv')
 buro_balance = pd.read_csv('.. /input/bureau_balance.csv')
 credit_card = pd.read_csv('.. /input/credit_card_balance.csv')
 POS_CASH = pd.read_csv('.. /input/POS_CASH_balance.csv')
 payments = pd.read_csv('.. /input/installments_payments.csv')
 lgbm_submission = pd.read_csv('.. /input/sample_submission.csv' )
sub.to_csv('sub.csv',index=False )<set_options>
y = data["TARGET"]
del data["TARGET"]
img_size =(224, 224)
 img_array_list = []
cls_list = []
dir = '.. /input/train/PNEUMONIA'
img_list = glob.glob(dir + '/*.jpeg')
 for i in img_list:
img = load_img(i, color_mode='grayscale', target_size=(img_size))
img_array = img_to_array(img)/ 255
img_array_list.append(img_array)
 cls_list.append(1)
 dir = '.. /input/train/NORMAL'
img_list = glob.glob(dir + '/*.jpeg')
 for i in img_list:
img = load_img(i, color_mode='grayscale', target_size=(img_size))
img_array = img_to_array(img)/ 255
img_array_list.append(img_array)
 cls_list.append(0)
 X_train = np.array(img_array_list)
 y_train = np.array(cls_list)
 print(X_train.shape, y_train.shape )<train_model>
categorical_feature = [col for col in data.columns if data[col].dtype == 'object']
X_learn, X_valid, y_learn, y_valid = train_test_split(X_train, y_train, test_size=0.3, random_state=0)
 shape = X_learn.shape
X_learn = np.reshape(X_learn,(shape[0], shape[1] * shape[2] * shape[3]))
ros = RandomOverSampler(random_state=0)
 X_res, y_res = ros.fit_resample(X_learn, y_learn)
 X_res = np.reshape(X_res,(X_res.shape[0], shape[1], shape[2], shape[3]))
Y_res = to_categorical(y_res)
 Y_valid = to_categorical(y_valid)
 print('Train:', X_res.shape, Y_res.shape)
 print('Test:', X_valid.shape, Y_valid.shape )<choose_model_class>
one_hot_df = pd.concat([data, test])
 one_hot_df = pd.get_dummies(one_hot_df, columns=categorical_feature )
def auc(y_true, y_pred):
auc = tf.py_func(roc_auc_score,(y_true, y_pred), tf.double)
 return auc
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu', input_shape=(img_size[0], img_size[1], 1)))
 model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3)))
 model.add(Dropout(rate=0.25))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3)))
 model.add(Dropout(rate=0.25))
model.add(Flatten())
 model.add(Dense(units=128, activation='relu'))
model.add(Dropout(rate=0.25))
model.add(Dense(units=2, activation='softmax'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', auc])
 model.summary()<train_model>
data = one_hot_df.iloc[:data.shape[0],:]
test = one_hot_df.iloc[data.shape[0]:,]
model.fit(X_res, Y_res, batch_size=32, epochs=10)
 model.evaluate(X_valid, Y_valid, batch_size=32 )<train_model>
buro_grouped_size = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].size()
buro_grouped_max = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].max()
buro_grouped_min = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].min()
shape = X_train.shape
X_train = np.reshape(X_train,(shape[0], shape[1] * shape[2] * shape[3]))
ros = RandomOverSampler(random_state=0)
 X_res, y_res = ros.fit_resample(X_train, y_train)
 X_train = np.reshape(X_train,(shape[0], shape[1], shape[2], shape[3]))
X_res = np.reshape(X_res,(X_res.shape[0], shape[1], shape[2], shape[3]))
Y_res = to_categorical(y_res)
 print('Train:', X_res.shape, Y_res.shape )<train_model>
buro_counts = buro_balance.groupby("SK_ID_BUREAU")["STATUS"].value_counts(normalize=False)
 buro_counts_unstacked = buro_counts.unstack("STATUS")
model.fit(X_res, Y_res, batch_size=32, epochs=10 )<normalization>
buro_counts_unstacked.columns = ['STATUS_0', 'STATUS_1','STATUS_2','STATUS_3','STATUS_4','STATUS_5','STATUS_C','STATUS_X',]
buro_counts_unstacked["MONTHS_COUNT"] = buro_grouped_size
buro_counts_unstacked["MONTHS_MIN"] = buro_grouped_min
buro_counts_unstacked["MONTHS_MAX"] = buro_grouped_max
img_array_list = []
dir = '.. /input/test'
img_list = glob.glob(dir + '/*.jpeg')
 img_list.sort()
for i in img_list:
img = load_img(i, color_mode='grayscale', target_size=(img_size))
img_array = img_to_array(img)/ 255
img_array_list.append(img_array)
 X_test = np.array(img_array_list)
 print(X_test.shape )<predict_on_test>
buro = buro.join(buro_counts_unstacked, how='left', on='SK_ID_BUREAU' )
predict = model.predict(X_test)[:, 1]<save_to_csv>
prev_cat_features = [pcol for pcol in prev.columns if prev[pcol].dtype == "object"]
prev = pd.get_dummies(prev, columns=prev_cat_features)
 prev.head(10 )
submit = pd.read_csv('.. /input/sampleSubmission.csv')
 submit['pneumonia'] = predict
submit.to_csv('submission.csv', index=False )<import_modules>
avg_prev = prev.groupby("SK_ID_CURR" ).mean()
avg_prev.head(10 )
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer<define_variables>
cnt_prev = prev[["SK_ID_CURR", "SK_ID_PREV"]].groupby("SK_ID_CURR" ).count()
avg_prev["nb_app"] = cnt_prev["SK_ID_PREV"]
del avg_prev["SK_ID_PREV"]
root_path = '.. /input/homework4_v2/'
max_df = 0.95
min_df = 5
smooth_idf = True
sublinear_tf = True
alpha = 1
beta = 0.75
gamma = 0.15
rel_count = 5
nrel_count = 1
iters = 5<define_variables>
buro_cat_features = [bcol for bcol in buro.columns if buro[bcol].dtype == 'object']
buro = pd.get_dummies(buro, columns=buro_cat_features)
 avg_buro = buro.groupby('SK_ID_CURR' ).mean()
avg_buro['buro_count'] = buro[['SK_ID_BUREAU', 'SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
del avg_buro['SK_ID_BUREAU']
with open(root_path + 'doc_list.txt')as file:
doc_list = [line.rstrip() for line in file]
with open(root_path + 'query_list.txt')as file:
query_list = [line.rstrip() for line in file]<string_transform>
le = LabelEncoder()
POS_CASH['NAME_CONTRACT_STATUS'] = le.fit_transform(POS_CASH['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
nunique_status2.head(10 )
documents, queries = [], []
for doc_name in doc_list:
with open(root_path + 'Document/' + doc_name)as file:
doc = ' '.join([word for line in file.readlines() [3:] for word in line.split() [:-1]])
 documents.append(doc)
 for query_name in query_list:
with open(root_path + 'Query/' + query_name)as file:
query = ' '.join([word for line in file.readlines() for word in line.split() [:-1]])
 queries.append(query )<compute_train_metric>
POS_CASH['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
POS_CASH['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
POS_CASH.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True )
vectorizer = TfidfVectorizer(max_df=max_df, min_df=min_df,
smooth_idf=smooth_idf, sublinear_tf=sublinear_tf)
 doc_tfidfs = vectorizer.fit_transform(documents ).toarray()
query_vecs = vectorizer.transform(queries ).toarray()
cos_sim = cosine_similarity(query_vecs, doc_tfidfs)
 rankings = np.flip(cos_sim.argsort() , axis=1 )<compute_test_metric>
le =LabelEncoder()
credit_card['NAME_CONTRACT_STATUS'] = le.fit_transform(credit_card['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
credit_card['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
credit_card['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
credit_card.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True)
for _ in range(iters):
rel_vecs = doc_tfidfs[rankings[:, :rel_count]].mean(axis=1)
 nrel_vecs = doc_tfidfs[rankings[:, -nrel_count:]].mean(axis=1)
 query_vecs = alpha * query_vecs + beta * rel_vecs - gamma * nrel_vecs
cos_sim = cosine_similarity(query_vecs, doc_tfidfs)
 rankings = np.flip(cos_sim.argsort(axis=1), axis=1 )<save_to_csv>
avg_payments = payments.groupby("SK_ID_CURR" ).mean()
avg_payments2 = payments.groupby("SK_ID_CURR" ).max()
avg_payments3 = payments.groupby("SK_ID_CURR" ).min()
with open('submission.csv', mode='w')as file:
file.write('Query,RetrievedDocuments
')
 for query_name, ranking in zip(query_list, rankings):
ranked_docs = ' '.join([doc_list[idx] for idx in ranking])
 file.write('%s,%s
' %(query_name, ranked_docs))<import_modules>
del avg_payments["SK_ID_PREV"]
print(os.listdir(".. /input")) 
<load_from_csv>
data = data.merge(right=avg_prev.reset_index() , how="left", on="SK_ID_CURR")
 test = test.merge(right=avg_prev.reset_index() , how="left", on="SK_ID_CURR")
data = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv('.. /input/test.csv')
 ids = pd.read_csv('.. /input/sampleSubmission.csv')
 test.head(5 )<count_missing_values>
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR' )
data.isnull().any()<feature_engineering>
data = data.merge(right=POS_CASH.groupby("SK_ID_CURR" ).mean().reset_index() , how="left", on="SK_ID_CURR")
 test = test.merge(right=POS_CASH.groupby("SK_ID_CURR" ).mean().reset_index() , how="left", on="SK_ID_CURR" )
data.loc[data['TotalCharges'].isna() ] = 0<count_missing_values>
data = data.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
test.isnull().any()<feature_engineering>
data = data.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
test.loc[test['TotalCharges'].isna() ] = 0<prepare_x_and_y>
data = data.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR' )
X = data.drop(['customerID','Churn'], 1)
 x = pd.get_dummies(X)
 x = x.drop(['gender_0','Partner_0','Dependents_0','PaymentMethod_0','MultipleLines_0','PaperlessBilling_0','StreamingMovies_0','PhoneService_0','InternetService_0','DeviceProtection_0','TechSupport_0','OnlineSecurity_0','OnlineBackup_0'],1)
 y = pd.get_dummies(data.Churn)
 y = y.drop([0],1)
 teste = test.drop(['customerID'], 1)
 teste = pd.get_dummies(teste)
 teste = teste.drop(['gender_0','Partner_0','Dependents_0','PaymentMethod_0','MultipleLines_0','PaperlessBilling_0','StreamingMovies_0','PhoneService_0','InternetService_0','DeviceProtection_0','TechSupport_0','OnlineSecurity_0','OnlineBackup_0'],1)
 y.head(2)
<prepare_x_and_y>
data = data.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR' )
test_np = np.array(teste)
 X=np.array(x)
 Y=np.array(y )<split>
test = test[test.columns[data.isnull().mean() < 0.85]]
data = data[data.columns[data.isnull().mean() < 0.85]]
x_Train,x_Test,y_Train,y_Test=train_test_split(X,Y,test_size=0.30 )<normalization>
excluded_feats = ["SK_ID_CURR"]
features = [f for f in data.columns if f not in excluded_feats]
scaler = StandardScaler()
sdsfds = scaler.fit_transform(x_Train )<compute_train_metric>
folds = KFold(n_splits = 4, shuffle=True, random_state=546789)
 oof_preds = np.zeros(data.shape[0])
 sub_preds = np.zeros(test.shape[0] )
def score(model,x,y):
prob=model.predict_proba(x)
 prob = prob[:, 1]
auc = roc_auc_score(y, prob)
 print('AUC: {}
ROC_AUC: {}
{}'.format(model.score(x,y),auc,prob[:10]))<train_model>
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(data)) :
trn_x, trn_y = data[features].iloc[trn_idx], y.iloc[trn_idx]
val_x, val_y = data[features].iloc[val_idx], y.iloc[val_idx]
clf = XGBClassifier(
 objective="binary:logistic",
booster="gbtree",
eval_metric = "auc",
nthread = 4,
eta = 0.05,
gamma = 0,
max_depth = 6,
subsample=0.7,
colsample_bytree = 0.7,
colsample_bylevel = 0.675,
min_child_weight = 22,
alpha = 0,
random_state = 42,
nrounds = 2000 
)
 clf.fit(trn_x, trn_y, eval_set=[(trn_x, trn_y),(val_x, val_y)], verbose=10, early_stopping_rounds=30)
 oof_preds[val_idx] = clf.predict_proba(val_x)[:, 1]
sub_preds += clf.predict_proba(test[features])[:, 1] / folds.n_splits
print("Fold %2d AUC : %.6f" %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()
knn=KNeighborsClassifier(n_neighbors=3)
 asa = knn.fit(x_Train,y_Train[:,1])
 score(knn,x_Test,y_Test[:,1] )<predict_on_test>
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test['TARGET'] = sub_preds
<train_model><EOS> 
test[["SK_ID_CURR","TARGET"]].to_csv('xgb_submission_esi.csv', index=False, float_format='%.8f' )