Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<filter>
RS = 12357
ROUNDS = 315
print("Started")
 np.random.seed(RS)
 input_folder = '.. /input/'
def train_xgb(X, y, params):
print("Will train XGB for {} rounds, RandomSeed: {}".format(ROUNDS, RS))
x, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=RS)
 xg_train = xgb.DMatrix(x, label=y_train)
 xg_val = xgb.DMatrix(X_val, label=y_val)
 watchlist = [(xg_train, 'train'),(xg_val, 'eval')]
return xgb.train(params, xg_train, ROUNDS, watchlist)
 def predict_xgb(clr, X_test):
return clr.predict(xgb.DMatrix(X_test))
def create_feature_map(features):
outfile = open('xgb.fmap', 'w')
 i = 0
for feat in features:
outfile.write('{0}\t{1}\tq
'.format(i, feat))
i = i + 1
outfile.close()
leak_df[leak_df.pred1_l1p.isnull() ]<compute_train_metric>
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.11
params['max_depth'] = 5
params['silent'] = 1
params['seed'] = RS
df_train = pd.read_csv(input_folder + 'train.csv')
 df_test = pd.read_csv(input_folder + 'test.csv')
 print("Original data: X_train: {}, X_test: {}".format(df_train.shape, df_test.shape))
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )<compute_test_metric>
print("Features processing")
 def get_weight(count, eps=10000, min_count=2):
return 0 if count < min_count else 1 /(count + eps)
 train_qs = pd.Series(df_train['question1'].tolist() + df_train['question2'].tolist() ).astype(str)
 words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
stops = set(stopwords.words("english"))
N = 10
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p * leak_df['pred1'].values +(1.-p)* leak_df ['pred3'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
def word_shares(row):
q1 = set(str(row['question1'] ).lower().split())
 q1words = q1.difference(stops)
 if len(q1words)== 0:
return '0:0:0:0:0'
q2 = set(str(row['question2'] ).lower().split())
 q2words = q2.difference(stops)
 if len(q2words)== 0:
return '0:0:0:0:0'
q1stops = q1.intersection(stops)
 q2stops = q2.intersection(stops)
 shared_words = q1words.intersection(q2words)
 shared_weights = [weights.get(w, 0)for w in shared_words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R1 = np.sum(shared_weights)/ np.sum(total_weights)
 R2 = len(shared_words)/(len(q1words)+ len(q2words))
R31 = len(q1stops)/ len(q1words)
 R32 = len(q2stops)/ len(q2words)
 return '{}:{}:{}:{}:{}'.format(R1, R2, len(shared_words), R31, R32 )
best_weight = np.argmin(scores)* 1./N
print(scores.min() , best_weight )<compute_test_metric>
df = pd.concat([df_train, df_test])
 df['word_shares'] = df.apply(word_shares, axis=1, raw=True)
 x = pd.DataFrame()
x['word_match'] = df['word_shares'].apply(lambda x: float(x.split(':')[0]))
x['tfidf_word_match'] = df['word_shares'].apply(lambda x: float(x.split(':')[1]))
x['shared_count'] = df['word_shares'].apply(lambda x: float(x.split(':')[2]))
x['stops1_ratio'] = df['word_shares'].apply(lambda x: float(x.split(':')[3]))
x['stops2_ratio'] = df['word_shares'].apply(lambda x: float(x.split(':')[4]))
x['diff_stops_r'] = x['stops1_ratio'] - x['stops2_ratio']
x['len_q1'] = df['question1'].apply(lambda x: len(str(x)))
 x['len_q2'] = df['question2'].apply(lambda x: len(str(x)))
 x['diff_len'] = x['len_q1'] - x['len_q2']
x['len_char_q1'] = df['question1'].apply(lambda x: len(str(x ).replace(' ', '')))
 x['len_char_q2'] = df['question2'].apply(lambda x: len(str(x ).replace(' ', '')))
 x['diff_len_char'] = x['len_char_q1'] - x['len_char_q2']
x['len_word_q1'] = df['question1'].apply(lambda x: len(str(x ).split()))
x['len_word_q2'] = df['question2'].apply(lambda x: len(str(x ).split()))
x['diff_len_word'] = x['len_word_q1'] - x['len_word_q2']
x['avg_world_len1'] = x['len_char_q1'] / x['len_word_q1']
x['avg_world_len2'] = x['len_char_q2'] / x['len_word_q2']
x['diff_avg_word'] = x['avg_world_len1'] - x['avg_world_len2']
x['exactly_same'] =(df['question1'] == df['question2'] ).astype(int)
 x['duplicated'] = df.duplicated(['question1', 'question2'] ).astype(int )
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p *(best_weight * leak_df['pred1'].values +(1.-best_weight)* leak_df ['pred3'].values)+(1.-p)* leak_df ['pred2'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
feature_names = list(x.columns.values)
 create_feature_map(feature_names)
 print("Features: {}".format(feature_names))
x_train = x[:df_train.shape[0]]
x_test = x[df_train.shape[0]:]
y_train = df_train['is_duplicate'].values
del x, df_train
best_weight2 = np.argmin(scores)* 1./N
print(scores.min() , best_weight2)
<define_search_space>
if 1:
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
print("Oversampling started for proportion: {}".format(len(pos_train)/(len(pos_train)+ len(neg_train))))
p = 0.175
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -= 1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print("Oversampling done, new proportion: {}".format(len(pos_train)/(len(pos_train)+ len(neg_train))))
x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train
all_combinations = list(np.linspace(0.2,0.5,31))
all_combinations<import_modules>
print("Training data: X_train: {}, Y_train: {}, X_test: {}".format(x_train.shape, len(y_train), x_test.shape))
ROUNDS = 500
clr = train_xgb(x_train, y_train, params)
 preds = predict_xgb(clr, x_test)
 print(preds.shape )
import itertools<concatenate>
print("Translate for output type...")
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = preds
sub.to_csv("xgb_seed{}_n{}.csv".format(RS, ROUNDS), index=False)
 print("Features importances...")
 importance = clr.get_fscore(fmap='xgb.fmap')
 importance = sorted(importance.items() , key=operator.itemgetter(1))
ft = pd.DataFrame(importance, columns=['feature', 'fscore'])
 ft.plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(10, 25))
plt.gcf().savefig('features_importance.png')
 print('Clear.' )