Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<load_from_csv>
warnings.filterwarnings("ignore")
 %matplotlib inline
train_df = pd.read_csv(data_source + "train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(data_source + "depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)] 
<feature_engineering>
LABELS = ["isFraud"]
all_files = glob.glob(".. /input/lgmodels/*.csv")
 scores = np.zeros(len(all_files))
for i in range(len(all_files)) :
scores[i] = float('.'+all_files[i].split(".")[3] )
test_df["images"] = [np.array(load_img(data_source + "test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(test_df.index)] 
<feature_engineering>
top = scores.argsort() [::-1]
for i, f in enumerate(top):
print(i,scores[f],all_files[f] )
train_df["images"] = [np.array(load_img(data_source + "train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
outs = [pd.read_csv(all_files[f], index_col=0)for f in top]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "m" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
train_df["masks"] = [np.array(load_img(data_source + "train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<merge>
mean_corr = corr.mean()
mean_corr = mean_corr.sort_values(ascending=True)
 mean_corr = mean_corr[:6]
mean_corr
<concatenate>
m_gmean1 = 0
for n in mean_corr.index:
m_gmean1 += np.log(concat_sub[n])
 m_gmean1 = np.exp(m_gmean1/len(mean_corr))
all_arr = pd.concat([train_df,test_df],sort=False)
 all_arr.insert(0, 'orig_id', range(1,len(all_arr)+1))<merge>
rank = np.tril(corr.values,-1)
 rank[rank<0.92] = 1
m =(rank>0 ).sum() -(rank>0.97 ).sum()
m_gmean2, s = 0, 0
for n in range(m):
mx = np.unravel_index(rank.argmin() , rank.shape)
 w =(m-n)/m
m_gmean2 += w*(np.log(concat_sub.iloc[:,mx[0]])+np.log(concat_sub.iloc[:,mx[1]])) /2
s += w
rank[mx] = 1
m_gmean2 = np.exp(m_gmean2/s )
all_arr = pd.merge(all_arr,all_files_df,how = 'left',left_index=True, right_on = 'img_name')
<categorify>
top_mean = 0
s = 0
for n in [0,1,3,7,26]:
top_mean += concat_sub.iloc[:,n]*scores[top[n]]
s += scores[top[n]]
top_mean /= s
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
<split>
concat_sub['isFraud'] = m_gmean
concat_sub[['isFraud']].to_csv('stack_gmean.csv' )