Kaggle competition: Santander Value Prediction Challenge
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
DATA_DIR = '.. /input/'
train = pd.read_csv(DATA_DIR+"train.csv")
 test = pd.read_csv(DATA_DIR+"test.csv")
 transact_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
def get_beautiful_test(test):
test_rnd = np.round(test.iloc[:, 1:], 2)
 ugly_indexes = []
non_ugly_indexes = []
for idx in tqdm(range(len(test))):
if not np.all(
 test_rnd.iloc[idx, :].values==test.iloc[idx, 1:].values 
):
ugly_indexes.append(idx)
 else:
non_ugly_indexes.append(idx)
 print(len(ugly_indexes), len(non_ugly_indexes))
np.save('test_ugly_indexes', np.array(ugly_indexes))
np.save('test_non_ugly_indexes', np.array(non_ugly_indexes))
test = test.iloc[non_ugly_indexes].reset_index(drop=True)
 return test, non_ugly_indexes, ugly_indexes
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )<init_hyperparams>
test, non_ugly_indexes, ugly_indexes = get_beautiful_test(test)
 test["target"] = train["target"].mean()
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
gc.collect() ;
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f',
'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5',
'58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867',
'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7',
'1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535',
'fc99f9426', '91f701ba2', '0572565c2', '190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98'
]
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
CPU_CORES = 1
def _get_leak(df, cols, lag=0, verbose=False):

 series_str = df[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 series_shifted_str = df[cols].shift(lag+2, axis=1)[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 if verbose:
target_rows = series_shifted_str.progress_apply(lambda x: np.where(x == series_str)[0])
 else:
target_rows = series_shifted_str.apply(lambda x: np.where(x == series_str)[0])
 target_vals = target_rows.apply(lambda x: df.loc[x[0], cols[lag]] if len(x)==1 else 0)
 return target_vals
def fast_get_leak(df, cols, lag=0):
d1 = df[cols[:-lag-2]].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2 = df[cols[lag+2:]].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2['pred'] = df[cols[lag]]
d3 = d2[~d2.duplicated(['key'], keep=False)]
return d1.merge(d3, how='left', on='key' ).pred.fillna(0)
 def compiled_leak_result() :
max_nlags = len(cols)- 2
train_leak = train[["ID", "target"] + cols]
train_leak["compiled_leak"] = 0
train_leak["nonzero_mean"] = train[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_value_corrects = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 train_leak[c] = fast_get_leak(train_leak, cols, i)
 leaky_cols.append(c)
 train_leak = train.join(
 train_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)
 zeroleak = train_leak["compiled_leak"]==0
train_leak.loc[zeroleak, "compiled_leak"] = train_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(train_leak["compiled_leak"] > 0))
_correct_counts = sum(train_leak["compiled_leak"]==train_leak["target"])
 leaky_value_corrects.append(_correct_counts/leaky_value_counts[-1])
 print("Leak values found in train", leaky_value_counts[-1])
 print(
 "% of correct leaks values in train ",
leaky_value_corrects[-1] 
)
 tmp = train_leak.copy()
tmp.loc[zeroleak, "compiled_leak"] = tmp.loc[zeroleak, "nonzero_mean"]
scores.append(np.sqrt(mean_squared_error(y, np.log1p(tmp["compiled_leak"] ).fillna(14.49))))
print(
 'Score(filled with nonzero mean)',
scores[-1] 
)
 result = dict(
 score=scores,
leaky_count=leaky_value_counts,
leaky_correct=leaky_value_corrects, 
)
 return train_leak, result
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
result = pd.DataFrame.from_dict(result, orient='columns')
 result.T
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<prepare_x_and_y>
result.to_csv('train_leaky_stat.csv', index=False )
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values<concatenate>
best_score = np.min(result['score'])
 best_lag = np.argmin(result['score'])
 print('best_score', best_score, '
best_lag', best_lag )
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
def rewrite_compiled_leak(leak_df, lag):
leak_df["compiled_leak"] = 0
for i in range(lag):
c = "leaked_target_"+str(i)
 zeroleak = leak_df["compiled_leak"]==0
leak_df.loc[zeroleak, "compiled_leak"] = leak_df.loc[zeroleak, c]
return leak_df
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )<train_model>
leaky_cols = [c for c in train_leak.columns if 'leaked_target_' in c]
train_leak = rewrite_compiled_leak(train_leak, best_lag)
 train_leak[['ID']+leaky_cols+['compiled_leak']].head()
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )<save_to_csv>
train_res = train_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 train_res.to_csv('train_leak.csv', index=False )
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<set_options>
def compiled_leak_result_test(max_nlags):
test_leak = test[["ID", "target"] + cols]
test_leak["compiled_leak"] = 0
test_leak["nonzero_mean"] = test[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 test_leak[c] = fast_get_leak(test_leak, cols, i)
 leaky_cols.append(c)
 test_leak = test.join(
 test_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = test_leak["compiled_leak"]==0
test_leak.loc[zeroleak, "compiled_leak"] = test_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(test_leak["compiled_leak"] > 0))
print("Leak values found in test", leaky_value_counts[-1])
 result = dict(
 leaky_count=leaky_value_counts, 
)
 return test_leak, result
warnings.filterwarnings('ignore' )<load_from_csv>
test_leak, test_result = compiled_leak_result_test(max_nlags=38 )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<choose_model_class>
test_result = pd.DataFrame.from_dict(test_result, orient='columns')
 test_result.T
class SSMCGaussianMixture(object):
def __init__(self, n_features, n_categories, n_clusters=3):
self.n_features = n_features
self.n_categories = n_categories
self.n_clusters = n_clusters
self.mus = np.array([np.random.randn(n_features)]*n_categories*n_clusters)
 self.sigmas = np.array([np.eye(n_features)]*n_categories*n_clusters)
 self.pis = np.array([1/(n_categories*n_clusters)]*(n_categories*n_clusters))
def fit(self, X_train, y_train, X_test, threshold=0.00001, max_iter=100):
Z_train_mask = np.zeros(shape=(len(X_train), self.n_categories*self.n_clusters))
for i in range(len(y_train)) :
Z_train_mask[i, y_train[i]*self.n_clusters:(y_train[i]+1)*self.n_clusters]=1
Z_train = Z_train_mask/self.n_clusters
for i in range(max_iter):
Z_train = np.array([self.gamma(X_train, k)for k in range(self.n_categories*self.n_clusters)] ).T
Z_train *= Z_train_mask
Z_train /= Z_train.sum(axis=1, keepdims=True)
 Z_test = np.array([self.gamma(X_test, k)for k in range(self.n_categories*self.n_clusters)] ).T
Z_test /= Z_test.sum(axis=1, keepdims=True)
 datas = [X_train, Z_train, X_test, Z_test]
mus = np.array([self._est_mu(k, *datas)for k in range(self.n_categories*self.n_clusters)])
 sigmas = np.array([self._est_sigma(k, *datas)for k in range(self.n_categories*self.n_clusters)])
 pis = np.array([self._est_pi(k, *datas)for k in range(self.n_categories*self.n_clusters)])
 diff = max(np.max(np.abs(mus-self.mus)) ,
np.max(np.abs(sigmas-self.sigmas)) ,
np.max(np.abs(pis-self.pis)))
 sigmas += np.array([np.eye(self.n_features)]*self.n_categories*self.n_clusters)*0.00000001
self.mus = mus
self.sigmas = sigmas
self.pis = pis
if diff<threshold:
break
def predict_proba(self, X):
Z_pred = np.array([self.gamma(X, k)for k in range(self.n_categories*self.n_clusters)] ).T
Z_pred /= Z_pred.sum(axis=1, keepdims=True)
 Y_pred = np.zeros(shape=(len(X), self.n_categories))
for i in range(self.n_categories):
Y_pred[:,i] = Z_pred[:,i*self.n_clusters:(i+1)*self.n_clusters].sum(axis=1)
 return Y_pred
def gamma(self, X, k):
return stats.multivariate_normal.pdf(X, mean=self.mus[k], cov=self.sigmas[k])
 def _est_mu(self, k, X_train, Z_train, X_test, Z_test):
mu =(Z_train[:,k]@X_train + Z_test[:,k]@X_test ).T / \ 
(Z_train[:,k].sum() + Z_test[:,k].sum())
 return mu
def _est_sigma(self, k, X_train, Z_train, X_test, Z_test):
cmp1 =(X_train-self.mus[k] ).T@np.diag(Z_train[:,k])@(X_train-self.mus[k])
 cmp2 =(X_test-self.mus[k] ).T@np.diag(Z_test[:,k])@(X_test-self.mus[k])
 sigma =(cmp1+cmp2)/(Z_train[:,k].sum() + Z_test[:k].sum())
 return sigma
def _est_pi(self, k, X_train, Z_train, X_test, Z_test):
pi =(Z_train[:,k].sum() + Z_test[:,k].sum())/ \ 
(Z_train.sum() + Z_test.sum())
 return pi 
<train_model>
test_result.to_csv('test_leaky_stat.csv', index=False )
class BaseClassifier(object):
def __init__(self):
self.preprocess = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 def fit(self, X_train, y_train, X_test, cv_qda=2, cv_meta=2):
X_train_org = X_train
self.preprocess_tune(np.vstack([X_train, X_test]))
X_train = self.preprocess.transform(X_train)
 X_test = self.preprocess.transform(X_test)
 self.cgm = SSMCGaussianMixture(n_features=X_train.shape[1], n_categories=2)
 self.cgm.fit(X_train, y_train, X_test)
 def predict(self, X):
X = self.preprocess.transform(X)
 return self.cgm.predict_proba(X)[:,1]
def preprocess_tune(self, X):
self.preprocess.fit(X)
 def validation(self, X, y):
X = self.preprocess.transform(X)
 kf = KFold(n_splits=3, shuffle=True)
 scores = []
for train_index, test_index in kf.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
self.cgm.fit(X_train, y_train, X_test)
 y_pred = self.cgm.predict_proba(X_test)[:,1]
scores.append(roc_auc_score(y_test, y_pred))
self.score = np.array(scores ).mean()
print('validation score = ', self.score )<prepare_x_and_y>
test_leak = rewrite_compiled_leak(test_leak, best_lag)
 test_leak[['ID']+leaky_cols+["compiled_leak", "nonzero_mean"]].head()
df_train_sample = df_train[df_train['wheezy-copper-turtle-magic']==wcrms[3]]
X_train_sample = df_train_sample.drop(['id', 'target', 'wheezy-copper-turtle-magic'], axis=1 ).values
y_train_sample = df_train_sample['target'].values
df_test_sample = df_test[df_test['wheezy-copper-turtle-magic']==wcrms[3]]
X_test_sample = df_test_sample.drop(['id', 'wheezy-copper-turtle-magic'], axis=1 ).values<train_model>
test_res = test_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 test_res.to_csv('test_leak.csv', index=False )
bc = BaseClassifier()
bc.fit(X_train_sample, y_train_sample, X_test_sample )<predict_on_test>
test_leak.loc[test_leak["compiled_leak"]==0, "compiled_leak"] = test_leak.loc[test_leak["compiled_leak"]==0, "nonzero_mean"]
<train_model><EOS> 
sub = pd.read_csv(DATA_DIR+"test.csv", usecols=["ID"])
 sub["target"] = 0
sub.iloc[non_ugly_indexes, 1] = test_leak["compiled_leak"].values
sub.to_csv(f"non_fake_sub_lag_{best_lag}.csv", index=False)
 print(f"non_fake_sub_lag_{best_lag}.csv saved" )