Kaggle competition: Quora Question Pairs
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0 )<feature_engineering>
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )<merge>
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<feature_engineering>
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )<filter>
print('Total number of question pairs for testing: {}'.format(len(df_test)) )
leak_df[leak_df.pred1_l1p.isnull() ]<compute_train_metric>
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )<compute_test_metric>
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )
N = 10
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p * leak_df['pred1'].values +(1.-p)* leak_df ['pred3'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
best_weight = np.argmin(scores)* 1./N
print(scores.min() , best_weight )<compute_test_metric>
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p *(best_weight * leak_df['pred1'].values +(1.-best_weight)* leak_df ['pred3'].values)+(1.-p)* leak_df ['pred2'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R
best_weight2 = np.argmin(scores)* 1./N
print(scores.min() , best_weight2)
<define_variables>
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )
all_combinations = list(np.linspace(0.1,0.5,401))
all_combinations1 = [i for i in all_combinations if i > 0.37 and i < 0.4]
all_combinations2 = [i for i in all_combinations if i > 0.34 and i < 0.385]
all_combinations3 = [i for i in all_combinations if i > 0.19 and i < 0.22]
all_combinations<import_modules>
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values
import itertools<concatenate>
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train
l = [all_combinations1, all_combinations2, all_combinations3]
all_l = list(itertools.product(*l)) + list(itertools.product(*reversed(l)) )<define_variables>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )
filtered_combis = [l for l in all_l if l[0] + l[1] + l[2] > 0.95 and l[0] + l[1] + l[2] < 1]<compute_test_metric>
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )
<feature_engineering><EOS> 
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )