Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<categorify>
from fastai.imports import *
from fastai.structured import *
from fastai.column_data import *
from torch.nn import functional as F
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
col='application_type'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train['application_type_ta']=enc_train
X_test['application_type_ta']=enc_test 
<count_missing_values>
df_train = pd.read_feather('.. /input/home-credit-data-processing-for-neural-networks/tables_merged_train')
 df_test = pd.read_feather('.. /input/home-credit-data-processing-for-neural-networks/tables_merged_test' )
X_train['Yaxis'].isnull().sum()<categorify>
df_train.dtypes.value_counts()
X_train['Yaxis']=X_train['Yaxis'].replace([np.inf, -np.inf,np.nan], -9999)
 X_test['Yaxis']=X_test['Yaxis'].replace([np.inf, -np.inf,np.nan], -9999)
 X_train['Yaxis'].astype(str)
 X_test['Yaxis'].astype(str)
 col='Yaxis'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train['Yaxis_ta']=enc_train
X_test['Yaxis_ta']=enc_test
X_train['Yaxis'].astype(int)
 X_test['Yaxis'].astype(int )<data_type_conversions>
cat_vars = [col for col in df_train if df_train[col].dtype.name != 'float64' and df_train[col].dtype.name != 'float32' and len(df_train[col].unique())< 150]
cat_vars.remove('TARGET' )
X_train=X_train.replace([np.inf, -np.inf,np.nan], -9999)
 X_test=X_test.replace([np.inf, -np.inf,np.nan], -9999)
<split>
cat_sz = [(c, len(df_train[c].unique())+1)for c in cat_vars]
scores = []
y_pred_test=np.zeros(len(X_test))
df = pd.DataFrame(index=[], columns=[])
 df['feature']=X_train.columns
n=10
for i in range(n):
X_train_,X_val,y_train_,y_val=train_test_split(X_train,y_train,test_size=0.3,random_state=i*10)
 clf = LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=100, n_jobs=-1, num_leaves=50, objective=None,
random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 clf.fit(X_train_, y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
score = roc_auc_score(y_val, y_pred)
 scores.append(score)
 df[i]=Series(clf.booster_.feature_importance(importance_type='gain'))
y_pred_test+=clf.predict_proba(X_test)[:,1]
df['ave']=df.mean(axis=1)
 df['std']=df.std(axis=1)
 df=df.sort_values('ave',ascending=False)
 ykai=y_pred_test/n 
<train_model>
y = np.array(df_train['TARGET'])
 df_train.drop('TARGET', axis = 1, inplace=True)
 df_to_nn_train, df_to_nn_valid, y_train, y_valid = train_test_split(df_train, y, test_size=0.33, random_state=23, stratify = y )
scores_xg=[]
y_pred_test_xg=np.zeros(len(X_test))
df_xg = pd.DataFrame(index=[], columns=[])
 df_xg['feature']=X_train.columns
n=10
for i in range(n):
X_train_,X_val,y_train_,y_val=train_test_split(X_train,y_train,test_size=0.3,random_state=i*10)
 xg=xgb.XGBClassifier()
xg.fit(X_train_, y_train_,early_stopping_rounds=100, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred_xg = xg.predict_proba(X_val)[:,1]
score_xg = roc_auc_score(y_val, y_pred_xg)
 print(score_xg)
 scores_xg.append(score_xg)
 y_pred_test_xg+=xg.predict_proba(X_test)[:,1]
ykai_xg=y_pred_test_xg/n 
<train_model>
def preprocess_fast_ai(df_to_nn_train, df_to_nn_valid, cat_vars):
for v in cat_vars: df_to_nn_train[v] = df_to_nn_train[v].astype('category' ).cat.as_ordered()
apply_cats(df_to_nn_valid, df_to_nn_train)
 df, _, nas, mapper = proc_df(df_to_nn_train, do_scale=True, skip_flds=['SK_ID_CURR'])
 df_valid, _, nas, mapper = proc_df(df_to_nn_valid, do_scale=True, na_dict=nas, mapper=mapper, skip_flds=['SK_ID_CURR'])
 return df, df_valid
scores_cb = []
y_pred_test_cb=np.zeros(len(X_test))
df_cb = pd.DataFrame(index=[], columns=[])
 df_cb['feature']=X_train.columns
n=10
for i in range(n):
X_train_,X_val,y_train_,y_val=train_test_split(X_train,y_train,test_size=0.3,random_state=i*10)
 cb = catboost.CatBoostClassifier(eval_metric='AUC')
 cb.fit(X_train_, y_train_, early_stopping_rounds=200,eval_set=[(X_val, y_val)])
 y_pred_cb = cb.predict_proba(X_val)[:,1]
y_pred_test_cb+=cb.predict_proba(X_test)[:,1]
ykai_cb=y_pred_test_cb/n<find_best_model_class>
%time df, df_valid = preprocess_fast_ai(df_to_nn_train, df_to_nn_valid, cat_vars )
<define_variables>
emb_szs = [(c, min(50,(c+1)//2)) for _,c in cat_sz]
y_pred=(ykai+ykai_xg+ykai_cb)/3 
<save_to_csv>
md = ColumnarModelData.from_data_frames('', trn_df = df, val_df = df_valid,
trn_y = y_train.astype('int'), val_y = y_valid.astype('int'),
cat_flds=cat_vars, bs=512, is_reg= False )
submission = pd.read_csv('.. /input/homework-for-students3/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )<load_from_csv>
class MixedInputModel(nn.Module):
def __init__(self, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
y_range=None, use_bn=False, is_reg=True, is_multi=False):
super().__init__()
self.embs = nn.ModuleList([nn.Embedding(c, s)for c,s in emb_szs])
 for emb in self.embs: emb_init(emb)
 n_emb = sum(e.embedding_dim for e in self.embs)
 self.n_emb, self.n_cont= n_emb, n_cont
szs = [n_emb + n_cont] + szs
self.lins = nn.ModuleList([
nn.Linear(szs[i], szs[i+1])for i in range(len(szs)-1)])
 self.bns = nn.ModuleList([
nn.BatchNorm1d(sz)for sz in szs[1:]])
 for o in self.lins: kaiming_normal(o.weight.data)
 self.outp = nn.Linear(szs[-1], out_sz)
 kaiming_normal(self.outp.weight.data)
 self.emb_drop = nn.Dropout(emb_drop)
 self.drops = nn.ModuleList([nn.Dropout(drop)for drop in drops])
 self.bn = nn.BatchNorm1d(n_cont)
 self.use_bn,self.y_range = use_bn,y_range
self.is_reg = is_reg
self.is_multi = is_multi
def forward(self, x_cat, x_cont):
x = []
for i,e in enumerate(self.embs):
x.append(e(x_cat[:,i]))
x = torch.cat(x, 1)
 x = self.emb_drop(x)
 x2 = self.bn(x_cont)
 x = torch.cat([x, x2], 1)
 for l,d,b in zip(self.lins, self.drops, self.bns):
x = F.relu(l(x))
if self.use_bn: x = b(x)
 x = d(x)
 x = self.outp(x)
 x = F.log_softmax(x)
 return x
df_train = pd.read_csv('.. /input/train.csv')
 df_valid = pd.read_csv('.. /input/valid.csv')
 df_sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<drop_column>
m = MixedInputModel(emb_szs, n_cont = len(df.columns)-len(cat_vars),
emb_drop = 0.05, out_sz = 2, szs = [500, 250, 250], drops = [0.1, 0.1, 0.1],
y_range = None, use_bn = False, is_reg = False, is_multi = False)
 bm = BasicModel(m.cuda() , 'binary_classifier' )
df_train.drop(['article_link'], axis=1)
 df_valid.drop(['article_link'], axis=1 )<import_modules>
class StructuredLearner(Learner):
def __init__(self, data, models, **kwargs):
super().__init__(data, models, **kwargs)
 self.crit = F.nll_loss
learn = StructuredLearner(md, bm )
from sklearn.feature_extraction.text import TfidfVectorizer<feature_engineering>
learn.lr_find(1e-4, 1)
 learn.sched.plot(100 )
TfidfVec = TfidfVectorizer()<create_dataframe>
lr = 1e-1
learn.fit(lr, 3, metrics=[roc_auc_own] )
all_headlines = pd.DataFrame()
all_headlines = pd.concat([df_train, df_valid] )<feature_engineering>
logpreds = learn.predict()
preds = np.exp(logpreds[:,1] )
Tfidf_vectorized_data = TfidfVec.fit_transform(all_headlines.headline )<split>
print(classification_report(y_valid,
preds_binary,
target_names= ['0', '1']))
df_train_vec = Tfidf_vectorized_data[:18696]
df_valid_vec = Tfidf_vectorized_data[18696:]<prepare_x_and_y>
class ColumnarDataset(Dataset):
def __init__(self, cats, conts, y, is_reg, is_multi):
n = len(cats[0])if cats else len(conts[0])
 self.cats = np.stack(cats, 1 ).astype(np.int64)if cats else np.zeros(( n,1))
self.conts = np.stack(conts, 1 ).astype(np.float32)if conts else np.zeros(( n,1))
self.y = np.zeros(( n,1)) if y is None else y
if is_reg:
self.y = self.y[:,None]
self.is_reg = is_reg
self.is_multi = is_multi
def __len__(self): return len(self.y)
 def __getitem__(self, idx):
return [self.cats[idx], self.conts[idx], self.y[idx]]
@classmethod
def from_data_frames(cls, df_cat, df_cont, y=None, is_reg=True, is_multi=False):
cat_cols = [c.values for n,c in df_cat.items() ]
cont_cols = [c.values for n,c in df_cont.items() ]
return cls(cat_cols, cont_cols, y, is_reg, is_multi)
 @classmethod
def from_data_frame(cls, df, cat_flds, y=None, is_reg=False, is_multi=False):
return cls.from_data_frames(df[cat_flds], df.drop(cat_flds, axis=1), y, is_reg, is_multi)
 class ColumnarModelData(ModelData):
def __init__(self, path, trn_ds, val_ds, bs, test_ds=None, shuffle=True):
test_dl = DataLoader(test_ds, bs, shuffle=False, num_workers=1)if test_ds is not None else None
super().__init__(path, DataLoader(trn_ds, bs, shuffle=shuffle, num_workers=1),
DataLoader(val_ds, bs*2, shuffle=False, num_workers=1), test_dl)
 @classmethod
def from_data_frames(cls, path, trn_df, trn_y, cat_flds, bs, val_df = None, val_y = None, is_reg = False, is_multi = False, test_df=None):
trn_ds = ColumnarDataset.from_data_frame(trn_df, cat_flds, trn_y, is_reg, is_multi)
 val_ds = ColumnarDataset.from_data_frame(val_df, cat_flds, val_y, is_reg, is_multi)if val_df is not None else None
test_ds = ColumnarDataset.from_data_frame(test_df, cat_flds, None, is_reg, is_multi)if test_df is not None else None
return cls(path, trn_ds, val_ds, bs, test_ds=test_ds)
 @classmethod
def from_data_frame(cls, path, val_idxs, df, y, cat_flds, bs, is_reg=True, is_multi=False, test_df=None): 
(( val_df, trn_df),(val_y, trn_y)) = split_by_idx(val_idxs, df, y)
 return cls.from_data_frames(path, trn_df, val_df, trn_y, val_y, cat_flds, bs, is_reg, is_multi, test_df=test_df )
y_train = df_train.is_sarcastic<import_modules>
train_ids = df_train['SK_ID_CURR']
test_ids = df_test['SK_ID_CURR']
%time train_df, test_df = preprocess_fast_ai(df_train, df_test, cat_vars )
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score<split>
ros = RandomOverSampler()
df_resampled, y_resampled = ros.fit_sample(df, y_train)
 df_resampled = pd.DataFrame(df_resampled, columns = df.columns)
 y_valid.mean() , y_resampled.mean()
df_train_1, df_train_2 = train_test_split(df_train_vec, test_size=0.1)
 df_y_1, df_y_2 = train_test_split(y_train, test_size=0.1 )<choose_model_class>
md = ColumnarModelData.from_data_frames('', trn_df = df_resampled,
val_df = df_valid, trn_y = y_resampled.astype('int'),
val_y = y_valid.astype('int'), cat_flds=cat_vars, bs=1024, is_reg = False,
test_df = test_df)
 class StructuredLearner(Learner):
def __init__(self, data, models, **kwargs):
super().__init__(data, models, **kwargs)
 self.crit = F.nll_loss
m = MixedInputModel(emb_szs, n_cont = len(df.columns)-len(cat_vars),
emb_drop = 0.4, out_sz = 2, szs = [1000, 500],
drops = [0.6, 0.6],y_range = None, use_bn = False, is_reg = False)
 bm = BasicModel(m.cuda() , 'binary_classifier')
 learn = StructuredLearner(md, bm )
model = SGDClassifier(n_jobs=-1, loss='hinge', random_state=42 )<predict_on_test>
learn.lr_find(1e-2, 2)
 learn.sched.plot(100 )
pred = model.predict(df_valid_vec )<save_to_csv>
lr = 0.1
learn.fit(lr, 3, metrics=[roc_auc_own] )
my_submission=pd.DataFrame({'ID': df_valid['ID'], 'is_sarcastic': pred})
 my_submission.to_csv('fepas_submission_3.csv',index=False )<set_options>
learn.fit(lr, 2, metrics=[roc_auc_own], cycle_len=1, cycle_mult=2 )
%matplotlib inline
print(tf.config.experimental.list_physical_devices('CPU'))
print(tf.config.experimental.list_physical_devices('GPU'))
print(tf.__version__ )<load_from_csv>
print(classification_report(y_valid,
preds_binary,
target_names= ['0', '1']))
false_positive_rate, true_positive_rate, threshold = roc_curve(y_valid,
preds_valid )
train_data = pd.read_csv('.. /input/bird-or-aircraft-dafe-open/train_x.csv', index_col=0, header=None)
 train_labels = pd.read_csv('.. /input/bird-or-aircraft-dafe-open/train_y.csv', index_col=0)
 test_data = pd.read_csv('.. /input/bird-or-aircraft-dafe-open/test_x.csv', index_col=0, header=None )<count_values>
logpreds = learn.predict(True)
 preds = np.exp(logpreds[:,1])
 submission = pd.DataFrame({'SK_ID_CURR': df_test['SK_ID_CURR'],
'TARGET': preds})
 submission.to_csv('submission.csv', index=False, float_format='%.8f' )
train_labels['target'].value_counts()<data_type_conversions>
m=learn.model
m.cuda()
train_data = train_data.to_numpy()
test_data = test_data.to_numpy()
train_labels = train_labels.to_numpy()<train_model>
def get_embeddings(embs, dataframe, ids, cat_vars):
embeddings = np.concatenate([to_np(embs[i](V(dataframe[cat_vars[i]])))for i in range(len(embs)) ], axis = 1)
 embedding_columns = ["embedding_"+str(i)for i in range(embeddings.shape[1])]
embedding_df = pd.DataFrame(embeddings, columns=embedding_columns)
 embedding_df = pd.concat([embedding_df, ids], axis = 1)
 return embedding_df
<choose_model_class><EOS> 
train_embeddings = get_embeddings(m.embs, train_df, train_ids, cat_vars)
 test_embeddings = get_embeddings(m.embs, test_df, test_ids, cat_vars)
 train_embeddings.to_csv('train_embeddings.csv', index=False)
 test_embeddings.to_csv('test_embeddings.csv', index=False )